agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v50 1/8] Make index_concurrently_create_copy more general 3+ messages / 2 participants [nested] [flat]
* [PATCH v50 1/8] Make index_concurrently_create_copy more general @ 2026-03-24 18:02 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Álvaro Herrera @ 2026-03-24 18:02 UTC (permalink / raw) Add a 'boolean concurrent' option, and make it work for both cases. Also rename it to index_create_copy. This allows it to be reused for other purposes -- specifically, for REPACK CONCURRENTLY. With the CONCURRENTLY option, REPACK cannot simply swap the heap file and rebuild its indexes. Instead, it needs to build a separate set of indexes (including system catalog entries) *before* the actual swap, to reduce the time AccessExclusiveLock needs to be held for. This approach is different from what CREATE INDEX CONCURRENTLY does. Per a suggestion from Mihail Nikalayeu. Author: Antonin Houska <[email protected]> Discussion: https://postgr.es/m/41104.1754922120@localhost --- src/backend/catalog/index.c | 41 ++++++++++++++++++++++---------- src/backend/commands/indexcmds.c | 9 +++---- src/include/catalog/index.h | 7 +++--- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 1ccfa687f05..e418d67e8e4 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -1289,17 +1289,17 @@ index_create(Relation heapRelation, } /* - * index_concurrently_create_copy + * index_create_copy * - * Create concurrently an index based on the definition of the one provided by - * caller. The index is inserted into catalogs and needs to be built later - * on. This is called during concurrent reindex processing. + * Create an index based on the definition of the one provided by caller. The + * index is inserted into catalogs. If 'concurrently' is TRUE, it needs to be + * built later on; otherwise it's built immediately. * * "tablespaceOid" is the tablespace to use for this index. */ Oid -index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, - Oid tablespaceOid, const char *newName) +index_create_copy(Relation heapRelation, bool concurrently, + Oid oldIndexId, Oid tablespaceOid, const char *newName) { Relation indexRelation; IndexInfo *oldInfo, @@ -1318,6 +1318,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, List *indexColNames = NIL; List *indexExprs = NIL; List *indexPreds = NIL; + int flags = 0; indexRelation = index_open(oldIndexId, RowExclusiveLock); @@ -1328,7 +1329,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, * Concurrent build of an index with exclusion constraints is not * supported. */ - if (oldInfo->ii_ExclusionOps != NULL) + if (oldInfo->ii_ExclusionOps != NULL && concurrently) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("concurrent index creation for exclusion constraints is not supported"))); @@ -1384,9 +1385,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, } /* - * Build the index information for the new index. Note that rebuild of - * indexes with exclusion constraints is not supported, hence there is no - * need to fill all the ii_Exclusion* fields. + * Build the index information for the new index. */ newInfo = makeIndexInfo(oldInfo->ii_NumIndexAttrs, oldInfo->ii_NumIndexKeyAttrs, @@ -1395,11 +1394,24 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, indexPreds, oldInfo->ii_Unique, oldInfo->ii_NullsNotDistinct, - false, /* not ready for inserts */ - true, + !concurrently, /* isready */ + concurrently, /* concurrent */ indexRelation->rd_indam->amsummarizing, oldInfo->ii_WithoutOverlaps); + /* fetch exclusion constraint info if any */ + if (indexRelation->rd_index->indisexclusion) + { + /* + * XXX Beware: we're making newInfo point to oldInfo-owned memory. It + * would be more orthodox to palloc+memcpy, but we don't need that + * here at present. + */ + newInfo->ii_ExclusionOps = oldInfo->ii_ExclusionOps; + newInfo->ii_ExclusionProcs = oldInfo->ii_ExclusionProcs; + newInfo->ii_ExclusionStrats = oldInfo->ii_ExclusionStrats; + } + /* * Extract the list of column names and the column numbers for the new * index information. All this information will be used for the index @@ -1436,6 +1448,9 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, stattargets[i].isnull = isnull; } + if (concurrently) + flags = INDEX_CREATE_SKIP_BUILD | INDEX_CREATE_CONCURRENT; + /* * Now create the new index. * @@ -1459,7 +1474,7 @@ index_concurrently_create_copy(Relation heapRelation, Oid oldIndexId, indcoloptions->values, stattargets, reloptionsDatum, - INDEX_CREATE_SKIP_BUILD | INDEX_CREATE_CONCURRENT, + flags, 0, true, /* allow table to be a system catalog? */ false, /* is_internal? */ diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 373e8234794..cba379810c7 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -3989,10 +3989,11 @@ ReindexRelationConcurrently(const ReindexStmt *stmt, Oid relationOid, const Rein tablespaceid = indexRel->rd_rel->reltablespace; /* Create new index definition based on given index */ - newIndexId = index_concurrently_create_copy(heapRel, - idx->indexId, - tablespaceid, - concurrentName); + newIndexId = index_create_copy(heapRel, + true, + idx->indexId, + tablespaceid, + concurrentName); /* * Now open the relation of the new index, a session-level lock is diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index a38e95bc0eb..ed9e4c37d27 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -101,10 +101,9 @@ extern Oid index_create(Relation heapRelation, #define INDEX_CONSTR_CREATE_REMOVE_OLD_DEPS (1 << 4) #define INDEX_CONSTR_CREATE_WITHOUT_OVERLAPS (1 << 5) -extern Oid index_concurrently_create_copy(Relation heapRelation, - Oid oldIndexId, - Oid tablespaceOid, - const char *newName); +extern Oid index_create_copy(Relation heapRelation, bool concurrently, + Oid oldIndexId, Oid tablespaceOid, + const char *newName); extern void index_concurrently_build(Oid heapRelationId, Oid indexRelationId); -- 2.47.3 --brnevsqjnuzpyok4 Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v50-0002-Rename-cluster.c-h-repack.c-h.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. @ 2026-06-22 07:34 Antonin Houska <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Antonin Houska @ 2026-06-22 07:34 UTC (permalink / raw) The default values may be needed by the executor when processing the concurrent data changes. In particular, ExecInsertIndexTuples() needs it when determining the value of the 'indexUnchanged' hint for the index AM. Like in copy_index_constraints(), we make the new catalog entries dependent on the transient relation, so they are dropped along with it automatically. --- src/backend/commands/repack.c | 97 +++++++++++++++++++ .../injection_points/specs/repack.spec | 3 +- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 4d177c868bb..725efa236bc 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -48,6 +48,7 @@ #include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" +#include "catalog/pg_attrdef.h" #include "catalog/pg_constraint.h" #include "catalog/pg_inherits.h" #include "catalog/toasting.h" @@ -204,6 +205,7 @@ static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHea static List *build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes); static void copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id); +static void copy_attribute_defaults(Relation old_heap, Relation new_heap); static Relation process_single_relation(RepackStmt *stmt, LOCKMODE lockmode, bool isTopLevel, @@ -1083,6 +1085,13 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose, Assert(CheckRelationOidLockedByMe(OIDNewHeap, AccessExclusiveLock, false)); NewHeap = table_open(OIDNewHeap, NoLock); + /* + * Copy attribute defaults - the executor may need them, in order to + * process the concurrent data changes. In particular, this is related to + * ExecInsertIndexTuples(). + */ + copy_attribute_defaults(OldHeap, NewHeap); + /* Copy the heap data into the new table in the desired order */ copy_table_data(NewHeap, OldHeap, index, snapshot, verbose, &swap_toast_by_content, &frozenXid, &cutoffMulti); @@ -3434,6 +3443,94 @@ copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id) CommandCounterIncrement(); } +/* + * Create a transient copy of attribute defaults for the transient table. + * + * Like above, the executor needs information on attribute defaults. Once the + * repacking is finished, the catalog entries we create here are dropped. + */ +static void +copy_attribute_defaults(Relation old_heap, Relation new_heap) +{ + Oid old_heap_id = RelationGetRelid(old_heap); + Oid new_heap_id = RelationGetRelid(new_heap); + ScanKeyData skey; + Relation rel; + Relation att_rel = NULL; + TupleDesc desc; + SysScanDesc scan; + HeapTuple tup; + ObjectAddress objrel; + + rel = table_open(AttrDefaultRelationId, RowExclusiveLock); + ObjectAddressSet(objrel, RelationRelationId, new_heap_id); + + ScanKeyInit(&skey, + Anum_pg_attrdef_adrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(old_heap_id)); + scan = systable_beginscan(rel, AttrDefaultIndexId, true, + NULL, 1, &skey); + desc = RelationGetDescr(rel); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_attrdef adform; + Oid oid; + Datum values[Natts_pg_attrdef] = {0}; + bool nulls[Natts_pg_attrdef] = {0}; + bool replaces[Natts_pg_attrdef] = {0}; + HeapTuple new_tup, att_tup, att_new_tup; + ObjectAddress objad; + Datum att_values[Natts_pg_attribute] = {0}; + bool att_nulls[Natts_pg_attribute] = {0}; + bool att_replaces[Natts_pg_attribute] = {0}; + + adform = (Form_pg_attrdef) GETSTRUCT(tup); + Assert(adform->adrelid == old_heap_id); + + oid = GetNewOidWithIndex(rel, AttrDefaultOidIndexId, + Anum_pg_attrdef_oid); + values[Anum_pg_attrdef_oid - 1] = ObjectIdGetDatum(oid); + replaces[Anum_pg_attrdef_oid - 1] = true; + values[Anum_pg_attrdef_adrelid - 1] = ObjectIdGetDatum(new_heap_id); + replaces[Anum_pg_attrdef_adrelid - 1] = true; + + new_tup = heap_modify_tuple(tup, desc, values, nulls, replaces); + + /* Insert it into the catalog. */ + CatalogTupleInsert(rel, new_tup); + + /* Create a dependency so it's removed when we drop the new heap. */ + ObjectAddressSet(objad, AttrDefaultRelationId, oid); + recordDependencyOn(&objad, &objrel, DEPENDENCY_AUTO); + + /* Set atthasdef - new heap has it cleared. */ + att_tup = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(new_heap_id), + ObjectIdGetDatum(adform->adnum)); + if (!HeapTupleIsValid(att_tup)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + adform->adnum, new_heap_id); + + att_values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(true); + att_replaces[Anum_pg_attribute_atthasdef - 1] = true; + + if (att_rel == NULL) + att_rel = table_open(AttributeRelationId, RowExclusiveLock); + att_new_tup = heap_modify_tuple(att_tup, RelationGetDescr(att_rel), + att_values, att_nulls, att_replaces); + ReleaseSysCache(att_tup); + CatalogTupleUpdate(att_rel, &att_new_tup->t_self, att_new_tup); + } + systable_endscan(scan); + + table_close(rel, RowExclusiveLock); + if (att_rel) + table_close(att_rel, RowExclusiveLock); + + CommandCounterIncrement(); +} + /* * Try to start a background worker to perform logical decoding of data * changes applied to relation while REPACK CONCURRENTLY is copying its diff --git a/src/test/modules/injection_points/specs/repack.spec b/src/test/modules/injection_points/specs/repack.spec index d727a9b056b..7896d1456ad 100644 --- a/src/test/modules/injection_points/specs/repack.spec +++ b/src/test/modules/injection_points/specs/repack.spec @@ -3,7 +3,8 @@ setup { CREATE EXTENSION injection_points; - CREATE TABLE repack_test(i int PRIMARY KEY, j int); + CREATE TABLE repack_test(i int PRIMARY KEY, j int, + k int GENERATED ALWAYS AS (j * 2) STORED); INSERT INTO repack_test(i, j) VALUES (1, 1), (2, 2), (3, 3), (4, 4); CREATE TABLE relfilenodes(node oid); -- 2.52.0 --=-=-=-- ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. @ 2026-06-22 07:34 Antonin Houska <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Antonin Houska @ 2026-06-22 07:34 UTC (permalink / raw) The default values may be needed by the executor when processing the concurrent data changes. In particular, ExecInsertIndexTuples() needs it when determining the value of the 'indexUnchanged' hint for the index AM. Like in copy_index_constraints(), we make the new catalog entries dependent on the transient relation, so they are dropped along with it automatically. --- src/backend/commands/repack.c | 97 +++++++++++++++++++ .../injection_points/specs/repack.spec | 3 +- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 4d177c868bb..725efa236bc 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -48,6 +48,7 @@ #include "catalog/namespace.h" #include "catalog/objectaccess.h" #include "catalog/pg_am.h" +#include "catalog/pg_attrdef.h" #include "catalog/pg_constraint.h" #include "catalog/pg_inherits.h" #include "catalog/toasting.h" @@ -204,6 +205,7 @@ static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHea static List *build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes); static void copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id); +static void copy_attribute_defaults(Relation old_heap, Relation new_heap); static Relation process_single_relation(RepackStmt *stmt, LOCKMODE lockmode, bool isTopLevel, @@ -1083,6 +1085,13 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose, Assert(CheckRelationOidLockedByMe(OIDNewHeap, AccessExclusiveLock, false)); NewHeap = table_open(OIDNewHeap, NoLock); + /* + * Copy attribute defaults - the executor may need them, in order to + * process the concurrent data changes. In particular, this is related to + * ExecInsertIndexTuples(). + */ + copy_attribute_defaults(OldHeap, NewHeap); + /* Copy the heap data into the new table in the desired order */ copy_table_data(NewHeap, OldHeap, index, snapshot, verbose, &swap_toast_by_content, &frozenXid, &cutoffMulti); @@ -3434,6 +3443,94 @@ copy_index_constraints(Relation old_index, Oid new_index_id, Oid new_heap_id) CommandCounterIncrement(); } +/* + * Create a transient copy of attribute defaults for the transient table. + * + * Like above, the executor needs information on attribute defaults. Once the + * repacking is finished, the catalog entries we create here are dropped. + */ +static void +copy_attribute_defaults(Relation old_heap, Relation new_heap) +{ + Oid old_heap_id = RelationGetRelid(old_heap); + Oid new_heap_id = RelationGetRelid(new_heap); + ScanKeyData skey; + Relation rel; + Relation att_rel = NULL; + TupleDesc desc; + SysScanDesc scan; + HeapTuple tup; + ObjectAddress objrel; + + rel = table_open(AttrDefaultRelationId, RowExclusiveLock); + ObjectAddressSet(objrel, RelationRelationId, new_heap_id); + + ScanKeyInit(&skey, + Anum_pg_attrdef_adrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(old_heap_id)); + scan = systable_beginscan(rel, AttrDefaultIndexId, true, + NULL, 1, &skey); + desc = RelationGetDescr(rel); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_attrdef adform; + Oid oid; + Datum values[Natts_pg_attrdef] = {0}; + bool nulls[Natts_pg_attrdef] = {0}; + bool replaces[Natts_pg_attrdef] = {0}; + HeapTuple new_tup, att_tup, att_new_tup; + ObjectAddress objad; + Datum att_values[Natts_pg_attribute] = {0}; + bool att_nulls[Natts_pg_attribute] = {0}; + bool att_replaces[Natts_pg_attribute] = {0}; + + adform = (Form_pg_attrdef) GETSTRUCT(tup); + Assert(adform->adrelid == old_heap_id); + + oid = GetNewOidWithIndex(rel, AttrDefaultOidIndexId, + Anum_pg_attrdef_oid); + values[Anum_pg_attrdef_oid - 1] = ObjectIdGetDatum(oid); + replaces[Anum_pg_attrdef_oid - 1] = true; + values[Anum_pg_attrdef_adrelid - 1] = ObjectIdGetDatum(new_heap_id); + replaces[Anum_pg_attrdef_adrelid - 1] = true; + + new_tup = heap_modify_tuple(tup, desc, values, nulls, replaces); + + /* Insert it into the catalog. */ + CatalogTupleInsert(rel, new_tup); + + /* Create a dependency so it's removed when we drop the new heap. */ + ObjectAddressSet(objad, AttrDefaultRelationId, oid); + recordDependencyOn(&objad, &objrel, DEPENDENCY_AUTO); + + /* Set atthasdef - new heap has it cleared. */ + att_tup = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(new_heap_id), + ObjectIdGetDatum(adform->adnum)); + if (!HeapTupleIsValid(att_tup)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + adform->adnum, new_heap_id); + + att_values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(true); + att_replaces[Anum_pg_attribute_atthasdef - 1] = true; + + if (att_rel == NULL) + att_rel = table_open(AttributeRelationId, RowExclusiveLock); + att_new_tup = heap_modify_tuple(att_tup, RelationGetDescr(att_rel), + att_values, att_nulls, att_replaces); + ReleaseSysCache(att_tup); + CatalogTupleUpdate(att_rel, &att_new_tup->t_self, att_new_tup); + } + systable_endscan(scan); + + table_close(rel, RowExclusiveLock); + if (att_rel) + table_close(att_rel, RowExclusiveLock); + + CommandCounterIncrement(); +} + /* * Try to start a background worker to perform logical decoding of data * changes applied to relation while REPACK CONCURRENTLY is copying its diff --git a/src/test/modules/injection_points/specs/repack.spec b/src/test/modules/injection_points/specs/repack.spec index d727a9b056b..7896d1456ad 100644 --- a/src/test/modules/injection_points/specs/repack.spec +++ b/src/test/modules/injection_points/specs/repack.spec @@ -3,7 +3,8 @@ setup { CREATE EXTENSION injection_points; - CREATE TABLE repack_test(i int PRIMARY KEY, j int); + CREATE TABLE repack_test(i int PRIMARY KEY, j int, + k int GENERATED ALWAYS AS (j * 2) STORED); INSERT INTO repack_test(i, j) VALUES (1, 1), (2, 2), (3, 3), (4, 4); CREATE TABLE relfilenodes(node oid); -- 2.52.0 --=-=-=-- ^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2026-06-22 07:34 UTC | newest] Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2026-03-24 18:02 [PATCH v50 1/8] Make index_concurrently_create_copy more general Álvaro Herrera <[email protected]> 2026-06-22 07:34 [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. Antonin Houska <[email protected]> 2026-06-22 07:34 [PATCH] Copy the relevant pg_attrdef catalog entries for the transient relation. Antonin Houska <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox