agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v4 1/2] Support changing a column into a stored generated column 96+ messages / 2 participants [nested] [flat]
* [PATCH v4 1/2] Support changing a column into a stored generated column @ 2026-05-14 21:51 Alberto Piai <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Alberto Piai @ 2026-05-14 21:51 UTC (permalink / raw) This adds an ALTER TABLE subcommand to turn a regular column into a stored generated column: ... ALTER COLUMN c ADD GENERATED ALWAYS as (expr) STORED The syntax is chosen to be similar to ... ALTER COLUMN ... ADD GENERATED ... AS IDENTITY with the difference that in this case, since we're dealing with a generated column, only ALWAYS is supported. Additionally, STORED must always be specified. This new operation fits together with SET EXPRESSION and DROP EXPRESSION. Phase 2 happens in the same pass as the former, in order to run the cleanup code in ATPostAlterTypeCleanup, without which for example we would not re-check constraints when rewriting the table. Partitioning/inheritance is supported in the same way as DROP EXPRESSION: it is allowed to change the whole inheritace tree to/from a generated column at once; it is forbidden to change the parent table ONLY and it is forbidden to change a partition directly. See 8bf6ec3ba3a44448817af47a080587f3b71bee08 and the associated discussion. There is one limitation: currently DROP EXPRESSION does not allow to change an inheritance tree of depth > 2. This seems like an oversight, but in order to not feature-creep this commit, this is postponed for later; it should then be fixed for both DROP EXPRESSION and this new command. This is mostly useful as a first step to be able to add a stored generated column without rewriting the table under an exclusive lock. For ease of review, the operation as of this commit always rewrites the contents of the column using the new generated expression. --- src/backend/commands/tablecmds.c | 194 ++++++++++++++++- src/backend/parser/gram.y | 31 +++ src/include/nodes/parsenodes.h | 1 + .../test_ddl_deparse/test_ddl_deparse.c | 3 + src/test/regress/expected/alter_table.out | 200 ++++++++++++++++++ src/test/regress/sql/alter_table.sql | 142 +++++++++++++ 6 files changed, 570 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 92b0f38c353..39faef0a114 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -783,6 +783,14 @@ static void ATExecMergePartitions(List **wqueue, AlteredTableInfo *tab, Relation static void ATExecSplitPartition(List **wqueue, AlteredTableInfo *tab, Relation rel, PartitionCmd *cmd, AlterTableUtilityContext *context); +static void ATPrepAddGeneratedAsExprStored(Relation rel, + AlterTableCmd *cmd, + bool recurse, bool recursing, + LOCKMODE lockmode); +static ObjectAddress ATExecAddGeneratedAsExprStored(AlteredTableInfo *tab, + Relation rel, + const char *colName, + Constraint *def); static List *collectPartitionIndexExtDeps(List *partitionOids); static void applyPartitionIndexExtDeps(Oid newPartOid, List *extDepState); static void freePartitionIndexExtDeps(List *extDepState); @@ -4769,6 +4777,7 @@ AlterTableGetLockLevel(List *cmds) case AT_AddIdentity: case AT_DropIdentity: case AT_SetIdentity: + case AT_AddGeneratedAsExprStored: case AT_SetExpression: case AT_DropExpression: case AT_SetCompression: @@ -5093,6 +5102,13 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode, context); pass = AT_PASS_SET_EXPRESSION; break; + case AT_AddGeneratedAsExprStored: + ATSimplePermissions(cmd->subtype, rel, + ATT_TABLE | ATT_PARTITIONED_TABLE | ATT_FOREIGN_TABLE); + ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode, context); + ATPrepAddGeneratedAsExprStored(rel, cmd, recurse, recursing, lockmode); + pass = AT_PASS_SET_EXPRESSION; + break; case AT_DropExpression: /* ALTER COLUMN DROP EXPRESSION */ ATSimplePermissions(cmd->subtype, rel, ATT_TABLE | ATT_PARTITIONED_TABLE | ATT_FOREIGN_TABLE); @@ -5487,6 +5503,12 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, case AT_SetExpression: address = ATExecSetExpression(tab, rel, cmd->name, cmd->def, lockmode); break; + case AT_AddGeneratedAsExprStored: + Assert(IsA(cmd->def, Constraint)); + address = ATExecAddGeneratedAsExprStored(tab, rel, + cmd->name, + (Constraint *) cmd->def); + break; case AT_DropExpression: address = ATExecDropExpression(rel, cmd->name, cmd->missing_ok, lockmode); break; @@ -6695,6 +6717,8 @@ alter_table_type_to_string(AlterTableType cmdtype) return "ALTER COLUMN ... SET NOT NULL"; case AT_SetExpression: return "ALTER COLUMN ... SET EXPRESSION"; + case AT_AddGeneratedAsExprStored: + return "ALTER COLUMN ... ADD GENERATED ALWAYS AS (...) STORED"; case AT_DropExpression: return "ALTER COLUMN ... DROP EXPRESSION"; case AT_SetStatistics: @@ -8851,6 +8875,172 @@ ATExecSetExpression(AlteredTableInfo *tab, Relation rel, const char *colName, return address; } +/* + * Preparation for + * + * ALTER TABLE ALTER COLUMN ADD GENERATED ALWAYS AS expr STORED + * + * Checks whether recursion is allowed, following the same logic as ATPrepDropExpression. + */ +static void +ATPrepAddGeneratedAsExprStored(Relation rel, + AlterTableCmd *cmd, + bool recurse, bool recursing, + LOCKMODE lockmode) +{ + /* + * Reject ONLY if there are child tables. See ATPrepDropExpression. + */ + if (!recurse && + find_inheritance_children(RelationGetRelid(rel), lockmode)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ALTER TABLE / ADD GENERATED ALWAYS AS (expr) STORED must be applied to child tables too"))); + + /* + * Cannot change only inherited columns to be stored generated columns. + */ + if (!recursing) + { + HeapTuple tuple; + Form_pg_attribute attTup; + + tuple = SearchSysCacheCopyAttName(RelationGetRelid(rel), cmd->name); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + cmd->name, RelationGetRelationName(rel)))); + + attTup = (Form_pg_attribute) GETSTRUCT(tuple); + + if (attTup->attinhcount > 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("cannot change inherited column to be a stored generated column"))); + } +} + +/* + * ALTER TABLE ALTER COLUMN ADD GENERATED ALWAYS AS expr STORED + */ +static ObjectAddress +ATExecAddGeneratedAsExprStored(AlteredTableInfo *tab, + Relation rel, + const char *colName, + Constraint *def) +{ + HeapTuple tuple; + Form_pg_attribute attTup; + AttrNumber attnum; + ObjectAddress address; + NewColumnValue *newval; + RawColumnDefault *rawEnt; + Relation pg_attribute; + List *newcons; + CookedConstraint *cookedDef; + + Assert(def->raw_expr != NULL); + Assert(def->cooked_expr == NULL); + Assert(def->generated_when == ATTRIBUTE_IDENTITY_ALWAYS); + Assert(def->generated_kind == ATTRIBUTE_GENERATED_STORED); + + tuple = SearchSysCacheAttName(RelationGetRelid(rel), colName); + if (!HeapTupleIsValid(tuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + colName, RelationGetRelationName(rel)))); + + attTup = (Form_pg_attribute) GETSTRUCT(tuple); + + attnum = attTup->attnum; + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter system column \"%s\"", + colName))); + + if (attTup->attgenerated) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column \"%s\" of relation \"%s\" is already a generated column", + colName, RelationGetRelationName(rel)))); + + /* + * Find everything that depends on the column (constraints, indexes, etc), + * and record enough information to let us recreate the objects. + */ + RememberAllDependentForRebuilding(tab, AT_AddGeneratedAsExprStored, + rel, attnum, colName); + + /* Mark as generated stored in pg_attribute */ + pg_attribute = table_open(AttributeRelationId, RowExclusiveLock); + attTup->attgenerated = ATTRIBUTE_GENERATED_STORED; + CatalogTupleUpdate(pg_attribute, &tuple->t_self, tuple); + table_close(pg_attribute, RowExclusiveLock); + + /* Make above changes visible */ + CommandCounterIncrement(); + + ReleaseSysCache(tuple); + + /* + * Remove previous default value, if any, and store the new generator + * expression. + */ + RemoveAttrDefault(RelationGetRelid(rel), attnum, DROP_RESTRICT, + false, false); + + rawEnt = palloc_object(RawColumnDefault); + rawEnt->attnum = attnum; + rawEnt->raw_default = def->raw_expr; + rawEnt->generated = def->generated_kind; + newcons = AddRelationNewConstraints(rel, list_make1(rawEnt), + NIL, false, true, false, NULL); + + /* Make above changes visible */ + CommandCounterIncrement(); + + /* + * At the moment, AddRelationNewConstraints always returns one element + * when called with a generated = STORED input, but guard against + * accessing an empty list anyway. + */ + if (list_length(newcons) < 1) + ereport(ERROR, + errmsg_internal("expected exactly one processed default value")); + + cookedDef = linitial(newcons); + + /* + * Clear all the missing values if we're rewriting the table, since this + * renders them pointless. + */ + RelationClearMissing(rel); + + /* Make above changes visible */ + CommandCounterIncrement(); + + /* Drop any pg_statistic entry for the column */ + RemoveStatistics(RelationGetRelid(rel), attnum); + + /* Schedule a rewrite */ + newval = palloc0_object(NewColumnValue); + newval->attnum = attnum; + newval->expr = (Expr *) cookedDef->expr; + newval->is_generated = true; + tab->newvals = lappend(tab->newvals, newval); + tab->rewrite |= AT_REWRITE_DEFAULT_VAL; + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), attnum); + + ObjectAddressSubSet(address, RelationRelationId, + RelationGetRelid(rel), attnum); + return address; +} + /* * ALTER TABLE ALTER COLUMN DROP EXPRESSION */ @@ -15320,7 +15510,9 @@ RememberAllDependentForRebuilding(AlteredTableInfo *tab, AlterTableType subtype, SysScanDesc scan; HeapTuple depTup; - Assert(subtype == AT_AlterColumnType || subtype == AT_SetExpression); + Assert(subtype == AT_AlterColumnType + || subtype == AT_SetExpression + || subtype == AT_AddGeneratedAsExprStored); depRel = table_open(DependRelationId, RowExclusiveLock); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index ff4e1388c55..8cd75572257 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -2725,6 +2725,37 @@ alter_table_cmd: n->name = $3; n->def = (Node *) c; + $$ = (Node *) n; + } + /* ALTER TABLE <name> ALTER [COLUMN] <colname> ADD GENERATED ALWAYS AS ( <expression> ) STORED */ + | ALTER opt_column ColId ADD_P GENERATED generated_when AS '(' a_expr ')' STORED + { + AlterTableCmd *n = makeNode(AlterTableCmd); + Constraint *c = makeNode(Constraint); + + c->contype = CONSTR_GENERATED; + c->generated_when = $6; + c->raw_expr = $9; + c->cooked_expr = NULL; + c->generated_kind = ATTRIBUTE_GENERATED_STORED; + c->location = @5; + + /* + * Like in the case of ColConstraintElem, we cannot handle + * this in the grammar because IDENTITY allows both ALWAYS + * and BY DEFAULT, while generated columns only allow + * ALWAYS. This would lead to shift/reduce conflicts. + */ + if (c->generated_when != ATTRIBUTE_IDENTITY_ALWAYS) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("for a generated column, GENERATED ALWAYS must be specified"), + parser_errposition(@6))); + + n->subtype = AT_AddGeneratedAsExprStored; + n->name = $3; + n->def = (Node *) c; + $$ = (Node *) n; } /* ALTER TABLE <name> ALTER [COLUMN] <colname> SET <sequence options>/RESET */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 91377a6cde3..620e2dd73bf 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2527,6 +2527,7 @@ typedef enum AlterTableType AT_CookedColumnDefault, /* add a pre-cooked column default */ AT_DropNotNull, /* alter column drop not null */ AT_SetNotNull, /* alter column set not null */ + AT_AddGeneratedAsExprStored, /* add generated always as (...) stored */ AT_SetExpression, /* alter column set expression */ AT_DropExpression, /* alter column drop expression */ AT_SetStatistics, /* alter column set statistics */ diff --git a/src/test/modules/test_ddl_deparse/test_ddl_deparse.c b/src/test/modules/test_ddl_deparse/test_ddl_deparse.c index 64a1dfa9f79..3132ceac61f 100644 --- a/src/test/modules/test_ddl_deparse/test_ddl_deparse.c +++ b/src/test/modules/test_ddl_deparse/test_ddl_deparse.c @@ -129,6 +129,9 @@ get_altertable_subcmdinfo(PG_FUNCTION_ARGS) case AT_SetNotNull: strtype = "SET NOT NULL"; break; + case AT_AddGeneratedAsExprStored: + strtype = "ADD GENERATED ALWAYS AS (...) STORED"; + break; case AT_SetExpression: strtype = "SET EXPRESSION"; break; diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 6dd22be0e8d..2d0ce414d12 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -4875,3 +4875,203 @@ drop publication pub1; drop schema alter1 cascade; drop schema alter2 cascade; NOTICE: drop cascades to table alter2.t1 +-- Tests for ALTER COLUMN ... ADD GENERATED ALWAYS as ( expr ) STORED +-- turning a regular column into a stored generated column +create schema testgen; +create table testgen.t1 (a int, b int not null); +insert into testgen.t1 (a, b) + select x, x from generate_series(1, 10) x; +alter table testgen.t1 alter column b + add generated always as (a * 2) stored; +\d+ testgen.t1 + Table "testgen.t1" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+------------------------------------+---------+--------------+------------- + a | integer | | | | plain | | + b | integer | | not null | generated always as (a * 2) stored | plain | | +Not-null constraints: + "t1_b_not_null" NOT NULL "b" + +select a, b, a * 2 as expected, b = (a * 2) as correct + from testgen.t1 order by a; + a | b | expected | correct +----+----+----------+--------- + 1 | 2 | 2 | t + 2 | 4 | 4 | t + 3 | 6 | 6 | t + 4 | 8 | 8 | t + 5 | 10 | 10 | t + 6 | 12 | 12 | t + 7 | 14 | 14 | t + 8 | 16 | 16 | t + 9 | 18 | 18 | t + 10 | 20 | 20 | t +(10 rows) + +insert into testgen.t1 (a, b) values (10, 20); +ERROR: cannot insert a non-DEFAULT value into column "b" +DETAIL: Column "b" is a generated column. +insert into testgen.t1 (a, b) values (10, 21); +ERROR: cannot insert a non-DEFAULT value into column "b" +DETAIL: Column "b" is a generated column. +drop table testgen.t1; +-- turning a regular column into a stored generated column +-- fails when another constraint conflicts with the new expression +create table testgen.t2 (a int, b int not null); +insert into testgen.t2 (a, b) select x, x * 2 from generate_series(0, 5) x; +alter table testgen.t2 add constraint chk_gen_clause check (b = a * 2); +select pg_relation_filenode('testgen.t2') as t2_filenode_before \gset +alter table testgen.t2 alter column b add generated always as (a * 3) stored; +ERROR: check constraint "chk_gen_clause" of relation "t2" is violated by some row +select pg_relation_filenode('testgen.t2') as t2_filenode_after \gset +select :t2_filenode_before = :t2_filenode_after as did_not_rewrite; + did_not_rewrite +----------------- + t +(1 row) + +\d+ testgen.t2 + Table "testgen.t2" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+---------+---------+--------------+------------- + a | integer | | | | plain | | + b | integer | | not null | | plain | | +Check constraints: + "chk_gen_clause" CHECK (b = (a * 2)) +Not-null constraints: + "t2_b_not_null" NOT NULL "b" + +drop table testgen.t2; +-- rewrite an indexed column +create table testgen.t3 (a int, b int); +create index idx_b on testgen.t3 (b); +insert into testgen.t3 (a, b) select x, x from generate_series(1, 10) x; +select pg_relation_filenode('testgen.idx_b') as idx_filenode_before \gset +alter table testgen.t3 alter column b add generated always as (a * 2) stored; +select pg_relation_filenode('testgen.idx_b') as idx_filenode_after \gset +select :idx_filenode_before != :idx_filenode_after as did_rewrite_idx; + did_rewrite_idx +----------------- + t +(1 row) + +drop table testgen.t3; +-- test support for partitioned tables and inheritance +create table testgen.tpart (a int, b int) partition by hash (a); +create table testgen.tpart_p1 partition of testgen.tpart + for values with (modulus 2, remainder 0); +create table testgen.tpart_p2 partition of testgen.tpart + for values with (modulus 2, remainder 1); +insert into testgen.tpart (a, b) select x, x from generate_series(1, 5) x; +-- altering the parent table, recursing +begin; +alter table testgen.tpart alter column b + add generated always as (a * 2) stored; +-- expected: all the partitions have been rewritten +select a, b, a * 2 as expected, b = (a * 2) as correct +from testgen.tpart_p1 order by a; + a | b | expected | correct +---+---+----------+--------- + 1 | 2 | 2 | t + 2 | 4 | 4 | t +(2 rows) + +select a, b, a * 2 as expected, b = (a * 2) as correct +from testgen.tpart_p2 order by a; + a | b | expected | correct +---+----+----------+--------- + 3 | 6 | 6 | t + 4 | 8 | 8 | t + 5 | 10 | 10 | t +(3 rows) + +rollback; +-- altering a single partition is not allowed +begin; +-- expected: error +alter table testgen.tpart_p1 alter column b + add generated always as (a * 2) stored; +ERROR: cannot change inherited column to be a stored generated column +rollback; +-- altering only the parent table is not allowed +begin; +-- expected: error +alter table only testgen.tpart alter column b + add generated always as (a * 2) stored; +ERROR: ALTER TABLE / ADD GENERATED ALWAYS AS (expr) STORED must be applied to child tables too +rollback; +drop table testgen.tpart; +-- subpartitions +create table testgen.tpart (a int, b int, c int) + partition by hash (a); +create table testgen.tpart_p1 partition of testgen.tpart + for values with (modulus 2, remainder 0) + partition by hash (b); +create table testgen.tpart_p1_1 partition of testgen.tpart_p1 + for values with (modulus 2, remainder 0); +create table testgen.tpart_p1_2 partition of testgen.tpart_p1 + for values with (modulus 2, remainder 1); +create table testgen.tpart_p2 partition of testgen.tpart + for values with (modulus 2, remainder 1) + partition by hash (b); +create table testgen.tpart_p2_1 partition of testgen.tpart_p2 + for values with (modulus 2, remainder 0); +create table testgen.tpart_p2_2 partition of testgen.tpart_p2 + for values with (modulus 2, remainder 1); +insert into testgen.tpart (a, b) +select x, y +from generate_series(1, 5) x + cross join generate_series(1, 5) y; +-- currently, it is not possible to change the generated state of an +-- inheritance tree of depth >= 2 (same as in DROP EXPRESSION), so we expect an +-- error here. This might be fixed later. +begin; +alter table testgen.tpart alter column c + add generated always as (a + b) stored; +ERROR: ALTER TABLE / ADD GENERATED ALWAYS AS (expr) STORED must be applied to child tables too +rollback; +drop table testgen.tpart; +-- tests for invalid invocations +alter table doesnotexist alter column foo + add generated always as (bar * 2) stored; +ERROR: relation "doesnotexist" does not exist +create table testgen.t1 (a int); +alter table testgen.t1 alter column doesnotexist + add generated always as (bar * 2) stored; +ERROR: column "doesnotexist" of relation "t1" does not exist +alter table testgen.t1 add column b int; +alter table testgen.t1 alter column b + add generated always as (doesnotexist * 2) stored; +ERROR: column "doesnotexist" does not exist +-- invalid: only supports ALWAYS +alter table testgen.t1 alter column b + add generated by default as (a * 2) stored; +ERROR: for a generated column, GENERATED ALWAYS must be specified +LINE 2: add generated by default as (a * 2) stored; + ^ +-- invalid: only supports STORED +alter table testgen.t1 alter column b add generated always as (a * 2); +ERROR: syntax error at or near ";" +LINE 1: ...e testgen.t1 alter column b add generated always as (a * 2); + ^ +alter table testgen.t1 alter column b add generated always as (a * 2) virtual; +ERROR: syntax error at or near "virtual" +LINE 1: ...n.t1 alter column b add generated always as (a * 2) virtual; + ^ +-- invalid: b is already a generated column +create table testgen.t2 (a int, b int generated always as (a * 2) stored); +alter table testgen.t2 alter column b add generated always as (a * 2) stored; +ERROR: column "b" of relation "t2" is already a generated column +drop table testgen.t2; +create table testgen.t3 (a int, b int); +-- invalid: expr must be immutable +alter table testgen.t3 alter column b + add generated always as (a + random()) stored; +ERROR: generation expression is not immutable +-- invalid: expr cannot use subselects +alter table testgen.t3 alter column b + add generated always as (a + (select 1)) stored; +ERROR: cannot use subquery in column generation expression +drop table testgen.t3; +drop schema testgen cascade; +NOTICE: drop cascades to table testgen.t1 diff --git a/src/test/regress/sql/alter_table.sql b/src/test/regress/sql/alter_table.sql index f5f13bbd3e7..51d818d4995 100644 --- a/src/test/regress/sql/alter_table.sql +++ b/src/test/regress/sql/alter_table.sql @@ -3159,3 +3159,145 @@ alter table alter1.t1 set schema alter2; drop publication pub1; drop schema alter1 cascade; drop schema alter2 cascade; + +-- Tests for ALTER COLUMN ... ADD GENERATED ALWAYS as ( expr ) STORED +-- turning a regular column into a stored generated column +create schema testgen; + +create table testgen.t1 (a int, b int not null); +insert into testgen.t1 (a, b) + select x, x from generate_series(1, 10) x; +alter table testgen.t1 alter column b + add generated always as (a * 2) stored; +\d+ testgen.t1 +select a, b, a * 2 as expected, b = (a * 2) as correct + from testgen.t1 order by a; +insert into testgen.t1 (a, b) values (10, 20); +insert into testgen.t1 (a, b) values (10, 21); +drop table testgen.t1; + +-- turning a regular column into a stored generated column +-- fails when another constraint conflicts with the new expression +create table testgen.t2 (a int, b int not null); +insert into testgen.t2 (a, b) select x, x * 2 from generate_series(0, 5) x; +alter table testgen.t2 add constraint chk_gen_clause check (b = a * 2); +select pg_relation_filenode('testgen.t2') as t2_filenode_before \gset +alter table testgen.t2 alter column b add generated always as (a * 3) stored; +select pg_relation_filenode('testgen.t2') as t2_filenode_after \gset +select :t2_filenode_before = :t2_filenode_after as did_not_rewrite; +\d+ testgen.t2 +drop table testgen.t2; + +-- rewrite an indexed column +create table testgen.t3 (a int, b int); +create index idx_b on testgen.t3 (b); +insert into testgen.t3 (a, b) select x, x from generate_series(1, 10) x; +select pg_relation_filenode('testgen.idx_b') as idx_filenode_before \gset +alter table testgen.t3 alter column b add generated always as (a * 2) stored; +select pg_relation_filenode('testgen.idx_b') as idx_filenode_after \gset +select :idx_filenode_before != :idx_filenode_after as did_rewrite_idx; +drop table testgen.t3; + +-- test support for partitioned tables and inheritance +create table testgen.tpart (a int, b int) partition by hash (a); +create table testgen.tpart_p1 partition of testgen.tpart + for values with (modulus 2, remainder 0); +create table testgen.tpart_p2 partition of testgen.tpart + for values with (modulus 2, remainder 1); +insert into testgen.tpart (a, b) select x, x from generate_series(1, 5) x; + +-- altering the parent table, recursing +begin; +alter table testgen.tpart alter column b + add generated always as (a * 2) stored; +-- expected: all the partitions have been rewritten +select a, b, a * 2 as expected, b = (a * 2) as correct +from testgen.tpart_p1 order by a; +select a, b, a * 2 as expected, b = (a * 2) as correct +from testgen.tpart_p2 order by a; +rollback; + +-- altering a single partition is not allowed +begin; +-- expected: error +alter table testgen.tpart_p1 alter column b + add generated always as (a * 2) stored; +rollback; + +-- altering only the parent table is not allowed +begin; +-- expected: error +alter table only testgen.tpart alter column b + add generated always as (a * 2) stored; +rollback; + +drop table testgen.tpart; + +-- subpartitions +create table testgen.tpart (a int, b int, c int) + partition by hash (a); +create table testgen.tpart_p1 partition of testgen.tpart + for values with (modulus 2, remainder 0) + partition by hash (b); +create table testgen.tpart_p1_1 partition of testgen.tpart_p1 + for values with (modulus 2, remainder 0); +create table testgen.tpart_p1_2 partition of testgen.tpart_p1 + for values with (modulus 2, remainder 1); +create table testgen.tpart_p2 partition of testgen.tpart + for values with (modulus 2, remainder 1) + partition by hash (b); +create table testgen.tpart_p2_1 partition of testgen.tpart_p2 + for values with (modulus 2, remainder 0); +create table testgen.tpart_p2_2 partition of testgen.tpart_p2 + for values with (modulus 2, remainder 1); +insert into testgen.tpart (a, b) +select x, y +from generate_series(1, 5) x + cross join generate_series(1, 5) y; +-- currently, it is not possible to change the generated state of an +-- inheritance tree of depth >= 2 (same as in DROP EXPRESSION), so we expect an +-- error here. This might be fixed later. +begin; +alter table testgen.tpart alter column c + add generated always as (a + b) stored; +rollback; + +drop table testgen.tpart; + +-- tests for invalid invocations +alter table doesnotexist alter column foo + add generated always as (bar * 2) stored; + +create table testgen.t1 (a int); + +alter table testgen.t1 alter column doesnotexist + add generated always as (bar * 2) stored; + +alter table testgen.t1 add column b int; + +alter table testgen.t1 alter column b + add generated always as (doesnotexist * 2) stored; + +-- invalid: only supports ALWAYS +alter table testgen.t1 alter column b + add generated by default as (a * 2) stored; + +-- invalid: only supports STORED +alter table testgen.t1 alter column b add generated always as (a * 2); +alter table testgen.t1 alter column b add generated always as (a * 2) virtual; + +-- invalid: b is already a generated column +create table testgen.t2 (a int, b int generated always as (a * 2) stored); +alter table testgen.t2 alter column b add generated always as (a * 2) stored; +drop table testgen.t2; + +create table testgen.t3 (a int, b int); +-- invalid: expr must be immutable +alter table testgen.t3 alter column b + add generated always as (a + random()) stored; +-- invalid: expr cannot use subselects +alter table testgen.t3 alter column b + add generated always as (a + (select 1)) stored; +drop table testgen.t3; + +drop schema testgen cascade; base-commit: 6b48f5d1a74168c78badfb2e59ef788bb8eb396e -- 2.47.0 --zfrmj4necy5zy3mo Content-Type: text/plain; charset=utf-8 Content-Disposition: attachment; filename="v4-0002-Try-to-avoid-a-rewrite-when-adding-a-stored-gener.patch" ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-06-16 11:54 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-06-16 11:54 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 221 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 160 insertions(+), 110 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index fa98b8b9247..04d39ad8c86 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -93,40 +93,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -173,22 +139,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -197,6 +160,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2613,18 +2579,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2633,7 +2612,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2682,14 +2661,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2705,7 +2684,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2718,7 +2697,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2726,7 +2705,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2735,6 +2714,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2742,16 +2723,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2763,10 +2743,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2794,8 +2774,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2944,10 +2924,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2958,9 +2939,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2968,13 +2949,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -2997,16 +2978,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3077,7 +3058,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3093,41 +3074,65 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); - chgcxt->cc_rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); - InitResultRelInfo(chgcxt->cc_rri, relation, 0, 0, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = (ResultRelInfo *) palloc(sizeof(ResultRelInfo)); + InitResultRelInfo(dest->rri, relation, 0, 0, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3137,12 +3142,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3161,34 +3166,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f9eb23e52c9..b4d5abbaca7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2660,6 +2660,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v01-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
* [PATCH 3/8] Introduce RepackDest structure. @ 2026-07-15 08:37 Antonin Houska <[email protected]> 0 siblings, 0 replies; 96+ messages in thread From: Antonin Houska @ 2026-07-15 08:37 UTC (permalink / raw) This is for ChangeContext to handle insertions into two relations: besides the new relation (whose file will eventually be used by the REPACKed relation), an "auxiliary relation" is needed sometimes, in order to get the data sorted. The concept is introduced and explained later in the patch series. --- src/backend/commands/repack.c | 223 ++++++++++++++++--------------- src/include/commands/repack.h | 48 +++++++ src/tools/pgindent/typedefs.list | 1 + 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 19927c15bf3..392332b4b2b 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -95,40 +95,6 @@ typedef struct Oid indexOid; } RelToCluster; -/* - * The first file exported by the decoding worker must contain a snapshot, the - * following ones contain the data changes. - */ -#define WORKER_FILE_SNAPSHOT 0 - -/* - * Information needed to apply concurrent data changes. - */ -typedef struct ChangeContext -{ - /* The relation the changes are applied to. */ - Relation cc_rel; - - /* Needed to update indexes of cc_rel. */ - ResultRelInfo *cc_rri; - EState *cc_estate; - - /* - * Existing tuples to UPDATE and DELETE are located via this index. We - * keep the scankey in partially initialized state to avoid repeated work. - * sk_argument is completed on the fly. - */ - Relation cc_ident_index; - ScanKey cc_ident_key; - int cc_ident_key_nentries; - - /* The latest column we need to deform to have the tuple identity */ - AttrNumber cc_last_key_attno; - - /* Sequential number of the file containing the changes. */ - int cc_file_seq; -} ChangeContext; - /* * Backend-local information to control the decoding worker. */ @@ -175,22 +141,19 @@ static List *get_tables_to_repack_partitioned(RepackCommand cmd, static bool repack_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid); -static void apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt); -static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt); -static void apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt); +static void apply_concurrent_changes(ChangeContext *chgcxt); +static void apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot); +static void apply_concurrent_update(RepackDest *dest, + TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple); static void apply_concurrent_delete(Relation rel, TupleTableSlot *slot); static void restore_tuple(BufFile *file, Relation relation, TupleTableSlot *slot); static void adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *src); -static bool find_target_tuple(Relation rel, ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved); -static bool identity_key_equal(ChangeContext *chgcxt, - TupleTableSlot *locator, +static bool identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate); static void process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, @@ -199,6 +162,9 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id); static void release_change_context(ChangeContext *chgcxt); +static void initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id); +static void release_change_dest(RepackDest *dest); static void rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, Oid identIdx, TransactionId frozenXid, @@ -2614,18 +2580,31 @@ RepackCommandAsString(RepackCommand cmd) } /* - * Apply all the changes stored in 'file'. + * Apply all the changes provided by decoding worker. */ static void -apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) +apply_concurrent_changes(ChangeContext *chgcxt) { ConcurrentChangeKind kind = '\0'; - Relation rel = chgcxt->cc_rel; + RepackDest *dest; + Relation rel; TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; bool have_old_tuple = false; MemoryContext oldcxt; + DecodingWorkerShared *shared; + char fname[MAXPGPATH]; + BufFile *file; + + dest = &chgcxt->cc_dest; + rel = dest->rel; + + shared = (DecodingWorkerShared *) dsm_segment_address(decoding_worker->seg); + + /* Open the file containing the changes. */ + DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); + file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2634,7 +2613,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); - oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(dest->estate)); while (true) { @@ -2683,14 +2662,14 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) if (kind == CHANGE_INSERT) { - apply_concurrent_insert(rel, spilled_tuple, chgcxt); + apply_concurrent_insert(dest, spilled_tuple); } else if (kind == CHANGE_DELETE) { bool found; /* Find the tuple to be deleted */ - found = find_target_tuple(rel, chgcxt, spilled_tuple, ondisk_tuple); + found = find_target_tuple(dest, spilled_tuple, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); apply_concurrent_delete(rel, ondisk_tuple); @@ -2706,7 +2685,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) key = spilled_tuple; /* Find the tuple to be updated or deleted. */ - found = find_target_tuple(rel, chgcxt, key, ondisk_tuple); + found = find_target_tuple(dest, key, ondisk_tuple); if (!found) elog(ERROR, "could not find target tuple"); @@ -2719,7 +2698,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) */ adjust_toast_pointers(rel, spilled_tuple, ondisk_tuple); - apply_concurrent_update(rel, spilled_tuple, ondisk_tuple, chgcxt); + apply_concurrent_update(dest, spilled_tuple, ondisk_tuple); ExecClearTuple(old_update_tuple); have_old_tuple = false; @@ -2727,7 +2706,7 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - ResetPerTupleExprContext(chgcxt->cc_estate); + ResetPerTupleExprContext(dest->estate); } /* Cleanup. */ @@ -2736,6 +2715,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) ExecDropSingleTupleTableSlot(old_update_tuple); MemoryContextSwitchTo(oldcxt); + + BufFileClose(file); } /* @@ -2743,16 +2724,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) * table. */ static void -apply_concurrent_insert(Relation rel, TupleTableSlot *slot, - ChangeContext *chgcxt) +apply_concurrent_insert(RepackDest *dest, TupleTableSlot *slot) { /* Put the tuple in the table, but make sure it won't be decoded */ - table_tuple_insert(rel, slot, GetCurrentCommandId(true), + table_tuple_insert(dest->rel, slot, GetCurrentCommandId(true), TABLE_INSERT_NO_LOGICAL, NULL); /* Update indexes with this new tuple. */ - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, 0, slot, NIL, NULL); @@ -2764,10 +2744,10 @@ apply_concurrent_insert(Relation rel, TupleTableSlot *slot, * table. */ static void -apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, - TupleTableSlot *ondisk_tuple, - ChangeContext *chgcxt) +apply_concurrent_update(RepackDest *dest, TupleTableSlot *spilled_tuple, + TupleTableSlot *ondisk_tuple) { + Relation rel = dest->rel; LockTupleMode lockmode; TM_FailureData tmfd; TU_UpdateIndexes update_indexes; @@ -2795,8 +2775,8 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, + ExecInsertIndexTuples(dest->rri, + dest->estate, flags, spilled_tuple, NIL, NULL); @@ -2948,10 +2928,11 @@ adjust_toast_pointers(Relation relation, TupleTableSlot *dest, TupleTableSlot *s * not found, return false. */ static bool -find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, +find_target_tuple(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *retrieved) { - Form_pg_index idx = chgcxt->cc_ident_index->rd_index; + Relation rel = dest->rel; + Form_pg_index idx = dest->ident_index->rd_index; IndexScanDesc scan; bool retval = false; @@ -2962,9 +2943,9 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * * Use the incoming tuple to finalize the scan key. */ - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; + ScanKey entry = &dest->ident_key[i]; AttrNumber attno = idx->indkey.values[i]; entry->sk_argument = locator->tts_values[attno - 1]; @@ -2972,13 +2953,13 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, } /* XXX no instrumentation for now */ - scan = index_beginscan(rel, chgcxt->cc_ident_index, GetActiveSnapshot(), - NULL, chgcxt->cc_ident_key_nentries, 0, 0); - index_rescan(scan, chgcxt->cc_ident_key, chgcxt->cc_ident_key_nentries, NULL, 0); + scan = index_beginscan(rel, dest->ident_index, GetActiveSnapshot(), + NULL, dest->ident_key_nentries, 0, 0); + index_rescan(scan, dest->ident_key, dest->ident_key_nentries, NULL, 0); while (index_getnext_slot(scan, ForwardScanDirection, retrieved)) { /* Be wary of temporal constraints */ - if (scan->xs_recheck && !identity_key_equal(chgcxt, locator, retrieved)) + if (scan->xs_recheck && !identity_key_equal(dest, locator, retrieved)) { CHECK_FOR_INTERRUPTS(); continue; @@ -3001,16 +2982,16 @@ find_target_tuple(Relation rel, ChangeContext *chgcxt, TupleTableSlot *locator, * used for temporal constraints. */ static bool -identity_key_equal(ChangeContext *chgcxt, TupleTableSlot *locator, +identity_key_equal(RepackDest *dest, TupleTableSlot *locator, TupleTableSlot *candidate) { - slot_getsomeattrs(locator, chgcxt->cc_last_key_attno); - slot_getsomeattrs(candidate, chgcxt->cc_last_key_attno); + slot_getsomeattrs(locator, dest->last_key_attno); + slot_getsomeattrs(candidate, dest->last_key_attno); - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + for (int i = 0; i < dest->ident_key_nentries; i++) { - ScanKey entry = &chgcxt->cc_ident_key[i]; - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + ScanKey entry = &dest->ident_key[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); @@ -3081,7 +3062,7 @@ process_concurrent_changes(XLogRecPtr end_of_wal, ChangeContext *chgcxt, bool do /* Open the file. */ DecodingWorkerFileName(fname, shared->relid, chgcxt->cc_file_seq); file = BufFileOpenFileSet(&shared->sfs.fs, fname, O_RDONLY, false); - apply_concurrent_changes(file, chgcxt); + apply_concurrent_changes(chgcxt); BufFileClose(file); @@ -3097,10 +3078,34 @@ static void initialize_change_context(ChangeContext *chgcxt, Relation relation, Oid ident_index_id) { - chgcxt->cc_rel = relation; + initialize_change_dest(&chgcxt->cc_dest, relation, ident_index_id); + + chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; +} + +/* + * Free up resources taken by a ChangeContext. + */ +static void +release_change_context(ChangeContext *chgcxt) +{ + release_change_dest(&chgcxt->cc_dest); +} + +/* + * Initialize the RepackDest struct for the given relation, with the given + * index as identity index. InvalidOid can be specified to only make the + * relation ready for insertions. + */ +static void +initialize_change_dest(RepackDest *dest, Relation relation, + Oid ident_index_id) +{ + dest->rel = relation; + dest->bistate = GetBulkInsertState(); /* Only initialize fields needed by ExecInsertIndexTuples(). */ - chgcxt->cc_estate = CreateExecutorState(); + dest->estate = CreateExecutorState(); /* * Set up a range table for the executor, containing our repacked table as @@ -3149,41 +3154,41 @@ initialize_change_context(ChangeContext *chgcxt, perminfo->updatedCols = updatedCols; /* finally we can initialize the range table proper */ - ExecInitRangeTable(chgcxt->cc_estate, list_make1(rte), perminfos, + ExecInitRangeTable(dest->estate, list_make1(rte), perminfos, bms_make_singleton(1)); } /* Set up our ResultRelInfo to use for index updates */ - chgcxt->cc_rri = makeNode(ResultRelInfo); - InitResultRelInfo(chgcxt->cc_rri, relation, 1, NULL, 0); - ExecOpenIndices(chgcxt->cc_rri, false); + dest->rri = makeNode(ResultRelInfo); + InitResultRelInfo(dest->rri, relation, 1, NULL, 0); + ExecOpenIndices(dest->rri, false); /* * The table's relcache entry already has the relcache entry for the * identity index; find that. */ - chgcxt->cc_ident_index = NULL; - for (int i = 0; i < chgcxt->cc_rri->ri_NumIndices; i++) + dest->ident_index = NULL; + for (int i = 0; i < dest->rri->ri_NumIndices; i++) { Relation ind_rel; - ind_rel = chgcxt->cc_rri->ri_IndexRelationDescs[i]; + ind_rel = dest->rri->ri_IndexRelationDescs[i]; if (ind_rel->rd_id == ident_index_id) { - chgcxt->cc_ident_index = ind_rel; + dest->ident_index = ind_rel; break; } } - if (chgcxt->cc_ident_index == NULL) + if (dest->ident_index == NULL) elog(ERROR, "could not find identity index"); /* Set up for scanning said identity index */ { Form_pg_index indexForm; - indexForm = chgcxt->cc_ident_index->rd_index; - chgcxt->cc_ident_key_nentries = indexForm->indnkeyatts; - chgcxt->cc_ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); + indexForm = dest->ident_index->rd_index; + dest->ident_key_nentries = indexForm->indnkeyatts; + dest->ident_key = (ScanKey) palloc_array(ScanKeyData, indexForm->indnkeyatts); for (int i = 0; i < indexForm->indnkeyatts; i++) { ScanKey entry; @@ -3193,12 +3198,12 @@ initialize_change_context(ChangeContext *chgcxt, opcode; StrategyNumber eq_strategy; - entry = &chgcxt->cc_ident_key[i]; + entry = &dest->ident_key[i]; - opfamily = chgcxt->cc_ident_index->rd_opfamily[i]; - opcintype = chgcxt->cc_ident_index->rd_opcintype[i]; + opfamily = dest->ident_index->rd_opfamily[i]; + opcintype = dest->ident_index->rd_opcintype[i]; eq_strategy = IndexAmTranslateCompareType(COMPARE_EQ, - chgcxt->cc_ident_index->rd_rel->relam, + dest->ident_index->rd_rel->relam, opfamily, false); if (eq_strategy == InvalidStrategy) elog(ERROR, "could not find equality strategy for index operator family %u for type %u", @@ -3217,34 +3222,30 @@ initialize_change_context(ChangeContext *chgcxt, i + 1, eq_strategy, opcode, (Datum) 0); - entry->sk_collation = chgcxt->cc_ident_index->rd_indcollation[i]; + entry->sk_collation = dest->ident_index->rd_indcollation[i]; } } /* Determine the last column we must deform to read the identity */ - chgcxt->cc_last_key_attno = InvalidAttrNumber; - for (int i = 0; i < chgcxt->cc_ident_key_nentries; i++) + dest->last_key_attno = InvalidAttrNumber; + for (int i = 0; i < dest->ident_key_nentries; i++) { - AttrNumber attno = chgcxt->cc_ident_index->rd_index->indkey.values[i]; + AttrNumber attno = dest->ident_index->rd_index->indkey.values[i]; Assert(attno > 0); - chgcxt->cc_last_key_attno = Max(chgcxt->cc_last_key_attno, attno); + dest->last_key_attno = Max(dest->last_key_attno, attno); } - - chgcxt->cc_file_seq = WORKER_FILE_SNAPSHOT + 1; } -/* - * Free up resources taken by a ChangeContext. - */ static void -release_change_context(ChangeContext *chgcxt) +release_change_dest(RepackDest *dest) { - ExecCloseIndices(chgcxt->cc_rri); - FreeExecutorState(chgcxt->cc_estate); + FreeBulkInsertState(dest->bistate); + ExecCloseIndices(dest->rri); + FreeExecutorState(dest->estate); /* XXX are these pfrees necessary? */ - pfree(chgcxt->cc_rri); - pfree(chgcxt->cc_ident_key); + pfree(dest->rri); + pfree(dest->ident_key); } /* diff --git a/src/include/commands/repack.h b/src/include/commands/repack.h index 27105c10591..07f887e99f6 100644 --- a/src/include/commands/repack.h +++ b/src/include/commands/repack.h @@ -16,6 +16,7 @@ #include <signal.h> #include "access/hio.h" +#include "access/skey.h" #include "nodes/execnodes.h" #include "nodes/parsenodes.h" #include "parser/parse_node.h" @@ -39,6 +40,53 @@ typedef struct ClusterParams extern PGDLLIMPORT volatile sig_atomic_t RepackMessagePending; +/* + * Table to apply concurrent data changes to. + */ +typedef struct RepackDest +{ + /* The relation the changes are applied to. */ + Relation rel; + + BulkInsertStateData *bistate; + + /* Needed to update indexes of cc_rel. */ + ResultRelInfo *rri; + EState *estate; + + /* + * Existing tuples to UPDATE and DELETE are located via this index. We + * keep the scankey in partially initialized state to avoid repeated work. + * sk_argument is completed on the fly. + */ + Relation ident_index; + ScanKey ident_key; + + int ident_key_nentries; + + /* The latest column we need to deform to have the tuple identity */ + AttrNumber last_key_attno; +} RepackDest; + +/* + * The first file exported by the decoding worker must contain a snapshot, the + * following ones contain the data changes. + */ +#define WORKER_FILE_SNAPSHOT 0 + +/* + * Information needed to apply concurrent data changes. + * + * XXX Now that it's in *.h file, rename to RepackChangeContext? + */ +typedef struct ChangeContext +{ + /* The destination table. */ + RepackDest cc_dest; + + /* Sequential number of the file containing the changes. */ + int cc_file_seq; +} ChangeContext; extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 56c1f997f88..25ac7079099 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2664,6 +2664,7 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepackCommand +RepackDest RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -- 2.52.0 --=-=-= Content-Type: text/plain Content-Disposition: attachment; filename=v02-0004-Use-multiple-snapshots-to-copy-the-data.patch ^ permalink raw reply [nested|flat] 96+ messages in thread
end of thread, other threads:[~2026-07-15 08:37 UTC | newest] Thread overview: 96+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2026-05-14 21:51 [PATCH v4 1/2] Support changing a column into a stored generated column Alberto Piai <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-06-16 11:54 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]> 2026-07-15 08:37 [PATCH 3/8] Introduce RepackDest structure. Antonin Houska <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox