agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v29 10/11] Add regression tests for Incremental View Maintenance 77+ messages / 2 participants [nested] [flat]
* [PATCH v29 10/11] Add regression tests for Incremental View Maintenance @ 2021-03-10 02:11 Takuma Hoshiai <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Takuma Hoshiai @ 2021-03-10 02:11 UTC (permalink / raw) --- src/bin/pg_dump/pg_dump.h | 1 + .../regress/expected/incremental_matview.out | 1030 +++++++++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/incremental_matview.sql | 533 +++++++++ 4 files changed, 1565 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/incremental_matview.out create mode 100644 src/test/regress/sql/incremental_matview.sql diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 3705891d25..71ea246abd 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -323,6 +323,7 @@ typedef struct _tableInfo int numParents; /* number of (immediate) parent tables */ struct _tableInfo **parents; /* TableInfos of immediate parents */ + bool isivm; /* is incrementally maintainable materialized view? */ /* diff --git a/src/test/regress/expected/incremental_matview.out b/src/test/regress/expected/incremental_matview.out new file mode 100644 index 0000000000..8946d09f5d --- /dev/null +++ b/src/test/regress/expected/incremental_matview.out @@ -0,0 +1,1030 @@ +-- create a table to use as a basis for views and materialized views in various combinations +CREATE TABLE mv_base_a (i int, j int); +INSERT INTO mv_base_a VALUES + (1,10), + (2,20), + (3,30), + (4,40), + (5,50); +CREATE TABLE mv_base_b (i int, k int); +INSERT INTO mv_base_b VALUES + (1,101), + (2,102), + (3,103), + (4,104); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_1 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) WITH NO DATA; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +ERROR: materialized view "mv_ivm_1" has not been populated +HINT: Use the REFRESH MATERIALIZED VIEW command. +REFRESH MATERIALIZED VIEW mv_ivm_1; +NOTICE: could not create an index on materialized view "mv_ivm_1" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 10 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 +(4 rows) + +-- REFRESH WITH NO DATA +BEGIN; +CREATE FUNCTION dummy_ivm_trigger_func() RETURNS TRIGGER AS $$ + BEGIN + RETURN NULL; + END +$$ language plpgsql; +CREATE CONSTRAINT TRIGGER dummy_ivm_trigger AFTER INSERT +ON mv_base_a FROM mv_ivm_1 FOR EACH ROW +EXECUTE PROCEDURE dummy_ivm_trigger_func(); +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; + count +------- + 17 +(1 row) + +REFRESH MATERIALIZED VIEW mv_ivm_1 WITH NO DATA; +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; + count +------- + 1 +(1 row) + +ROLLBACK; +-- immediate maintenance +BEGIN; +INSERT INTO mv_base_b VALUES(5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 10 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 + 5 | 50 | 105 +(5 rows) + +UPDATE mv_base_a SET j = 0 WHERE i = 1; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 0 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 + 5 | 50 | 105 +(5 rows) + +DELETE FROM mv_base_b WHERE (i,k) = (5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 0 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 +(4 rows) + +ROLLBACK; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 10 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 +(4 rows) + +-- rename of IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_rename AS SELECT DISTINCT * FROM mv_base_a; +NOTICE: created index "mv_ivm_rename_index" on materialized view "mv_ivm_rename" +ALTER MATERIALIZED VIEW mv_ivm_rename RENAME COLUMN __ivm_count__ TO xxx; +ERROR: IVM column can not be renamed +DROP MATERIALIZED VIEW mv_ivm_rename; +-- unique index on IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_unique AS SELECT DISTINCT * FROM mv_base_a; +NOTICE: created index "mv_ivm_unique_index" on materialized view "mv_ivm_unique" +CREATE UNIQUE INDEX ON mv_ivm_unique(__ivm_count__); +ERROR: unique index creation on IVM columns is not supported +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__)); +ERROR: unique index creation on IVM columns is not supported +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__ + 1)); +ERROR: unique index creation on IVM columns is not supported +DROP MATERIALIZED VIEW mv_ivm_unique; +-- TRUNCATE a base table in join views +BEGIN; +TRUNCATE mv_base_a; +SELECT * FROM mv_ivm_1; + i | j | k +---+---+--- +(0 rows) + +ROLLBACK; +BEGIN; +TRUNCATE mv_base_b; +SELECT * FROM mv_ivm_1; + i | j | k +---+---+--- +(0 rows) + +ROLLBACK; +-- some query syntax +BEGIN; +CREATE FUNCTION ivm_func() RETURNS int LANGUAGE 'sql' + AS 'SELECT 1' IMMUTABLE; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_func AS SELECT * FROM ivm_func(); +NOTICE: could not create an index on materialized view "mv_ivm_func" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_no_tbl AS SELECT 1; +NOTICE: could not create an index on materialized view "mv_ivm_no_tbl" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +ROLLBACK; +-- result of materialized view have DISTINCT clause or the duplicate result. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_duplicate AS SELECT j FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_duplicate" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_distinct AS SELECT DISTINCT j FROM mv_base_a; +NOTICE: created index "mv_ivm_distinct_index" on materialized view "mv_ivm_distinct" +INSERT INTO mv_base_a VALUES(6,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; + j +---- + 10 + 20 + 20 + 30 + 40 + 50 +(6 rows) + +SELECT * FROM mv_ivm_distinct ORDER BY 1; + j +---- + 10 + 20 + 30 + 40 + 50 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; + j +---- + 10 + 20 + 30 + 40 + 50 +(5 rows) + +SELECT * FROM mv_ivm_distinct ORDER BY 1; + j +---- + 10 + 20 + 30 + 40 + 50 +(5 rows) + +ROLLBACK; +-- support SUM(), COUNT() and AVG() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(i), AVG(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 120 | 2 | 60.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +UPDATE mv_base_a SET j = 200 WHERE (i,j) = (2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+---------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 220 | 2 | 110.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) = (2,200); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +ROLLBACK; +-- support COUNT(*) aggregate function +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; + i | sum | count +---+-----+------- + 1 | 10 | 1 + 2 | 20 | 1 + 3 | 30 | 1 + 4 | 40 | 1 + 5 | 50 | 1 +(5 rows) + +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; + i | sum | count +---+-----+------- + 1 | 10 | 1 + 2 | 120 | 2 + 3 | 30 | 1 + 4 | 40 | 1 + 5 | 50 | 1 +(5 rows) + +ROLLBACK; +-- TRUNCATE a base table in aggregate views +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +TRUNCATE mv_base_a; +SELECT sum, count FROM mv_ivm_agg; + sum | count +-----+------- +(0 rows) + +SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; + i | sum | count +---+-----+------- +(0 rows) + +ROLLBACK; +-- support aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_group" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_group ORDER BY 1; + sum | count | avg +-----+-------+--------------------- + 150 | 5 | 30.0000000000000000 +(1 row) + +INSERT INTO mv_base_a VALUES(6,60); +SELECT * FROM mv_ivm_group ORDER BY 1; + sum | count | avg +-----+-------+--------------------- + 210 | 6 | 35.0000000000000000 +(1 row) + +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_group ORDER BY 1; + sum | count | avg +-----+-------+----- + | 0 | +(1 row) + +ROLLBACK; +-- TRUNCATE a base table in aggregate views without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_group" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +TRUNCATE mv_base_a; +SELECT sum, count, avg FROM mv_ivm_group; + sum | count | avg +-----+-------+----- + | 0 | +(1 row) + +SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; + sum | count | avg +-----+-------+----- + | 0 | +(1 row) + +ROLLBACK; +-- resolved issue: When use AVG() function and values is indivisible, result of AVG() is incorrect. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_avg_bug AS SELECT i, SUM(j), COUNT(j), AVG(j) FROM mv_base_A GROUP BY i; +NOTICE: created index "mv_ivm_avg_bug_index" on materialized view "mv_ivm_avg_bug" +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +INSERT INTO mv_base_a VALUES + (1,0), + (1,0), + (2,30), + (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 3 | 3.3333333333333333 + 2 | 80 | 3 | 26.6666666666666667 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) = (1,0); +DELETE FROM mv_base_a WHERE (i,j) = (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +ROLLBACK; +-- support MIN(), MAX() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT i, MIN(j), MAX(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_min_max_index" on materialized view "mv_ivm_min_max" +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; + i | min | max +---+-----+----- + 1 | 10 | 10 + 2 | 20 | 20 + 3 | 30 | 30 + 4 | 40 | 40 + 5 | 50 | 50 +(5 rows) + +INSERT INTO mv_base_a VALUES + (1,11), (1,12), + (2,21), (2,22), + (3,31), (3,32), + (4,41), (4,42), + (5,51), (5,52); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; + i | min | max +---+-----+----- + 1 | 10 | 12 + 2 | 20 | 22 + 3 | 30 | 32 + 4 | 40 | 42 + 5 | 50 | 52 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) IN ((1,10), (2,21), (3,32)); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; + i | min | max +---+-----+----- + 1 | 11 | 12 + 2 | 20 | 22 + 3 | 30 | 31 + 4 | 40 | 42 + 5 | 50 | 52 +(5 rows) + +ROLLBACK; +-- support MIN(), MAX() aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT MIN(j), MAX(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_min_max" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + 10 | 50 +(1 row) + +INSERT INTO mv_base_a VALUES + (0,0), (6,60), (7,70); +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + 0 | 70 +(1 row) + +DELETE FROM mv_base_a WHERE (i,j) IN ((0,0), (7,70)); +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + 10 | 60 +(1 row) + +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + | +(1 row) + +ROLLBACK; +-- Test MIN/MAX after search_path change +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min AS SELECT MIN(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_min" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_min; + min +----- + 10 +(1 row) + +CREATE SCHEMA myschema; +GRANT ALL ON SCHEMA myschema TO public; +CREATE TABLE myschema.mv_base_a (j int); +INSERT INTO myschema.mv_base_a VALUES (1); +DELETE FROM mv_base_a WHERE (i,j) = (1,10); +SELECT * FROM mv_ivm_min; + min +----- + 20 +(1 row) + +SET search_path TO myschema,public,pg_catalog; +DELETE FROM public.mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_min; + min +----- + 30 +(1 row) + +ROLLBACK; +-- aggregate views with column names specified +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; + a | sum +---+------ + 1 | 110 + 2 | 2200 + 3 | 300 + 4 | 40 + 5 | 50 +(5 rows) + +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; + a | b +---+------ + 1 | 110 + 2 | 2200 + 3 | 300 + 4 | 40 + 5 | 50 +(5 rows) + +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b,c) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +ERROR: too many column names were specified +ROLLBACK; +-- support self join view and multiple change on the same table +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1, 10), (2, 20), (3, 30); +CREATE INCREMENTAL MATERIALIZED VIEW mv_self(v1, v2) AS + SELECT t1.v, t2.v FROM base_t AS t1 JOIN base_t AS t2 ON t1.i = t2.i; +NOTICE: could not create an index on materialized view "mv_self" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_self ORDER BY v1; + v1 | v2 +----+---- + 10 | 10 + 20 | 20 + 30 | 30 +(3 rows) + +INSERT INTO base_t VALUES (4,40); +DELETE FROM base_t WHERE i = 1; +UPDATE base_t SET v = v*10 WHERE i=2; +SELECT * FROM mv_self ORDER BY v1; + v1 | v2 +-----+----- + 30 | 30 + 40 | 40 + 200 | 200 +(3 rows) + +WITH + ins_t1 AS (INSERT INTO base_t VALUES (5,50) RETURNING 1), + ins_t2 AS (INSERT INTO base_t VALUES (6,60) RETURNING 1), + upd_t AS (UPDATE base_t SET v = v + 100 RETURNING 1), + dlt_t AS (DELETE FROM base_t WHERE i IN (4,5) RETURNING 1) +SELECT NULL; + ?column? +---------- + +(1 row) + +SELECT * FROM mv_self ORDER BY v1; + v1 | v2 +-----+----- + 50 | 50 + 60 | 60 + 130 | 130 + 300 | 300 +(4 rows) + +--- with sub-transactions +SAVEPOINT p1; +INSERT INTO base_t VALUES (7,70); +RELEASE SAVEPOINT p1; +INSERT INTO base_t VALUES (7,77); +SELECT * FROM mv_self ORDER BY v1, v2; + v1 | v2 +-----+----- + 50 | 50 + 60 | 60 + 70 | 70 + 70 | 77 + 77 | 70 + 77 | 77 + 130 | 130 + 300 | 300 +(8 rows) + +ROLLBACK; +-- support simultaneous table changes +BEGIN; +CREATE TABLE base_r (i int, v int); +CREATE TABLE base_s (i int, v int); +INSERT INTO base_r VALUES (1, 10), (2, 20), (3, 30); +INSERT INTO base_s VALUES (1, 100), (2, 200), (3, 300); +CREATE INCREMENTAL MATERIALIZED VIEW mv(v1, v2) AS + SELECT r.v, s.v FROM base_r AS r JOIN base_s AS s USING(i); +NOTICE: could not create an index on materialized view "mv" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv ORDER BY v1; + v1 | v2 +----+----- + 10 | 100 + 20 | 200 + 30 | 300 +(3 rows) + +WITH + ins_r AS (INSERT INTO base_r VALUES (1,11) RETURNING 1), + ins_r2 AS (INSERT INTO base_r VALUES (3,33) RETURNING 1), + ins_s AS (INSERT INTO base_s VALUES (2,222) RETURNING 1), + upd_r AS (UPDATE base_r SET v = v + 1000 WHERE i = 2 RETURNING 1), + dlt_s AS (DELETE FROM base_s WHERE i = 3 RETURNING 1) +SELECT NULL; + ?column? +---------- + +(1 row) + +SELECT * FROM mv ORDER BY v1; + v1 | v2 +------+----- + 10 | 100 + 11 | 100 + 1020 | 200 + 1020 | 222 +(4 rows) + +ROLLBACK; +-- support foreign reference constraints +BEGIN; +CREATE TABLE ri1 (i int PRIMARY KEY); +CREATE TABLE ri2 (i int PRIMARY KEY REFERENCES ri1(i) ON UPDATE CASCADE ON DELETE CASCADE, v int); +INSERT INTO ri1 VALUES (1),(2),(3); +INSERT INTO ri2 VALUES (1),(2),(3); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ri(i1, i2) AS + SELECT ri1.i, ri2.i FROM ri1 JOIN ri2 USING(i); +NOTICE: created index "mv_ri_index" on materialized view "mv_ri" +SELECT * FROM mv_ri ORDER BY i1; + i1 | i2 +----+---- + 1 | 1 + 2 | 2 + 3 | 3 +(3 rows) + +UPDATE ri1 SET i=10 where i=1; +DELETE FROM ri1 WHERE i=2; +SELECT * FROM mv_ri ORDER BY i2; + i1 | i2 +----+---- + 3 | 3 + 10 | 10 +(2 rows) + +ROLLBACK; +-- views including NULL +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1,10),(2, NULL); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +NOTICE: could not create an index on materialized view "mv" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv ORDER BY i; + i | v +---+---- + 1 | 10 + 2 | +(2 rows) + +UPDATE base_t SET v = 20 WHERE i = 2; +SELECT * FROM mv ORDER BY i; + i | v +---+---- + 1 | 10 + 2 | 20 +(2 rows) + +ROLLBACK; +BEGIN; +CREATE TABLE base_t (i int); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +NOTICE: could not create an index on materialized view "mv" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv ORDER BY i; + i +--- +(0 rows) + +INSERT INTO base_t VALUES (1),(NULL); +SELECT * FROM mv ORDER BY i; + i +--- + 1 + +(2 rows) + +ROLLBACK; +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (1, 10), (1, 20); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, sum(v) FROM base_t GROUP BY i; +NOTICE: created index "mv_index" on materialized view "mv" +SELECT * FROM mv ORDER BY i; + i | sum +---+----- + 1 | 30 + | 3 +(2 rows) + +UPDATE base_t SET v = v * 10; +SELECT * FROM mv ORDER BY i; + i | sum +---+----- + 1 | 300 + | 30 +(2 rows) + +ROLLBACK; +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (NULL, 3), (NULL, 4), (NULL, 5); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, min(v), max(v) FROM base_t GROUP BY i; +NOTICE: created index "mv_index" on materialized view "mv" +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 1 | 5 +(1 row) + +DELETE FROM base_t WHERE v = 1; +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 2 | 5 +(1 row) + +DELETE FROM base_t WHERE v = 3; +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 2 | 5 +(1 row) + +DELETE FROM base_t WHERE v = 5; +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 2 | 4 +(1 row) + +ROLLBACK; +-- IMMV containing user defined type +BEGIN; +CREATE TYPE mytype; +CREATE FUNCTION mytype_in(cstring) + RETURNS mytype AS 'int4in' + LANGUAGE INTERNAL STRICT IMMUTABLE; +NOTICE: return type mytype is only a shell +CREATE FUNCTION mytype_out(mytype) + RETURNS cstring AS 'int4out' + LANGUAGE INTERNAL STRICT IMMUTABLE; +NOTICE: argument type mytype is only a shell +CREATE TYPE mytype ( + LIKE = int4, + INPUT = mytype_in, + OUTPUT = mytype_out +); +CREATE FUNCTION mytype_eq(mytype, mytype) + RETURNS bool AS 'int4eq' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_lt(mytype, mytype) + RETURNS bool AS 'int4lt' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_cmp(mytype, mytype) + RETURNS integer AS 'btint4cmp' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE OPERATOR = ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_eq); +CREATE OPERATOR < ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_lt); +CREATE OPERATOR CLASS mytype_ops + DEFAULT FOR TYPE mytype USING btree AS + OPERATOR 1 <, + OPERATOR 3 = , + FUNCTION 1 mytype_cmp(mytype,mytype); +CREATE TABLE t_mytype (x mytype); +CREATE INCREMENTAL MATERIALIZED VIEW mv_mytype AS + SELECT * FROM t_mytype; +NOTICE: could not create an index on materialized view "mv_mytype" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +INSERT INTO t_mytype VALUES ('1'::mytype); +SELECT * FROM mv_mytype; + x +--- + 1 +(1 row) + +ROLLBACK; +-- outer join is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv(a,b) AS SELECT a.i, b.i FROM mv_base_a a LEFT JOIN mv_base_b b ON a.i=b.i; +ERROR: OUTER JOIN is not supported on incrementally maintainable materialized view +-- CTE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv AS + WITH b AS ( SELECT * FROM mv_base_b) SELECT a.i,a.j FROM mv_base_a a, b WHERE a.i = b.i; +ERROR: CTE is not supported on incrementally maintainable materialized view +-- contain system column +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm01 AS SELECT i,j,xmin FROM mv_base_a; +ERROR: system column is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm02 AS SELECT i,j FROM mv_base_a WHERE xmin = '610'; +ERROR: system column is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT i,j,xmin::text AS x_min FROM mv_base_a; +ERROR: system column is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm06 AS SELECT i,j,xidsend(xmin) AS x_min FROM mv_base_a; +ERROR: system column is not supported on incrementally maintainable materialized view +-- contain subquery +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm03 AS SELECT i,j FROM mv_base_a WHERE i IN (SELECT i FROM mv_base_b WHERE k < 103 ); +ERROR: subquery is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT * FROM mv_base_b) b WHERE a.i = b.i; +ERROR: subquery is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm05 AS SELECT i,j, (SELECT k FROM mv_base_b b WHERE a.i = b.i) FROM mv_base_a a; +ERROR: subquery is not supported on incrementally maintainable materialized view +-- contain ORDER BY +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) ORDER BY i,j,k; +ERROR: ORDER BY clause is not supported on incrementally maintainable materialized view +-- contain HAVING +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) GROUP BY i,j,k HAVING SUM(i) > 5; +ERROR: HAVING clause is not supported on incrementally maintainable materialized view +-- contain view or materialized view +CREATE VIEW b_view AS SELECT i,k FROM mv_base_b; +CREATE MATERIALIZED VIEW b_mview AS SELECT i,k FROM mv_base_b; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT a.i,a.j FROM mv_base_a a,b_view b WHERE a.i = b.i; +ERROR: VIEW or MATERIALIZED VIEW is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT a.i,a.j FROM mv_base_a a,b_mview b WHERE a.i = b.i; +ERROR: VIEW or MATERIALIZED VIEW is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm09 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT i, COUNT(*) FROM mv_base_b GROUP BY i) b WHERE a.i = b.i; +ERROR: subquery is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm10 AS SELECT a.i,a.j FROM mv_base_a a WHERE EXISTS(SELECT 1 FROM mv_base_b b WHERE a.i = b.i) OR a.i > 5; +ERROR: subquery is not supported on incrementally maintainable materialized view +-- contain mutable functions +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm12 AS SELECT i,j FROM mv_base_a WHERE i = random()::int; +ERROR: mutable function is not supported on incrementally maintainable materialized view +HINT: functions must be marked IMMUTABLE +-- LIMIT/OFFSET is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm13 AS SELECT i,j FROM mv_base_a LIMIT 10 OFFSET 5; +ERROR: LIMIT/OFFSET clause is not supported on incrementally maintainable materialized view +-- DISTINCT ON is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm14 AS SELECT DISTINCT ON(i) i, j FROM mv_base_a; +ERROR: DISTINCT ON is not supported on incrementally maintainable materialized view +-- TABLESAMPLE clause is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm15 AS SELECT i, j FROM mv_base_a TABLESAMPLE SYSTEM(50); +ERROR: TABLESAMPLE clause is not supported on incrementally maintainable materialized view +-- window functions are not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm16 AS SELECT *, cume_dist() OVER (ORDER BY i) AS rank FROM mv_base_a; +ERROR: window functions are not supported on incrementally maintainable materialized view +-- aggregate function with some options is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm17 AS SELECT COUNT(*) FILTER(WHERE i < 3) FROM mv_base_a; +ERROR: aggregate function with FILTER clause is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm18 AS SELECT COUNT(DISTINCT i) FROM mv_base_a; +ERROR: aggregate function with DISTINCT arguments is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm19 AS SELECT array_agg(j ORDER BY i DESC) FROM mv_base_a; +ERROR: aggregate function with ORDER clause is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm20 AS SELECT i,SUM(j) FROM mv_base_a GROUP BY GROUPING SETS((i),()); +ERROR: GROUPING SETS, ROLLUP, or CUBE clauses is not supported on incrementally maintainable materialized view +-- inheritance parent is not supported +BEGIN; +CREATE TABLE parent (i int, v int); +CREATE TABLE child_a(options text) INHERITS(parent); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm21 AS SELECT * FROM parent; +ERROR: inheritance parent is not supported on incrementally maintainable materialized view +ROLLBACK; +-- UNION statement is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm22 AS SELECT i,j FROM mv_base_a UNION ALL SELECT i,k FROM mv_base_b;; +ERROR: UNION/INTERSECT/EXCEPT statements are not supported on incrementally maintainable materialized view +-- empty target list is not allowed with IVM +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm25 AS SELECT FROM mv_base_a; +ERROR: empty target list is not supported on incrementally maintainable materialized view +-- FOR UPDATE/SHARE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm26 AS SELECT i,j FROM mv_base_a FOR UPDATE; +ERROR: FOR UPDATE/SHARE clause is not supported on incrementally maintainable materialized view +-- tartget list cannot contain ivm column that start with '__ivm' +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm28 AS SELECT i AS "__ivm_count__" FROM mv_base_a; +ERROR: column name __ivm_count__ is not supported on incrementally maintainable materialized view +-- expressions specified in GROUP BY must appear in the target list. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm29 AS SELECT COUNT(i) FROM mv_base_a GROUP BY i; +ERROR: GROUP BY expression not appearing in select list is not supported on incrementally maintainable materialized view +-- experssions containing an aggregate is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm30 AS SELECT sum(i)*0.5 FROM mv_base_a; +ERROR: expression containing an aggregate in it is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm31 AS SELECT sum(i)/sum(j) FROM mv_base_a; +ERROR: expression containing an aggregate in it is not supported on incrementally maintainable materialized view +-- VALUES is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_only_values1 AS values(1); +ERROR: VALUES is not supported on incrementally maintainable materialized view +-- views containing base tables with Row Level Security +DROP USER IF EXISTS ivm_admin; +NOTICE: role "ivm_admin" does not exist, skipping +DROP USER IF EXISTS ivm_user; +NOTICE: role "ivm_user" does not exist, skipping +CREATE USER ivm_admin; +CREATE USER ivm_user; +--- create a table with RLS +SET SESSION AUTHORIZATION ivm_admin; +CREATE TABLE rls_tbl(id int, data text, owner name); +INSERT INTO rls_tbl VALUES + (1,'foo','ivm_user'), + (2,'bar','postgres'); +CREATE TABLE num_tbl(id int, num text); +INSERT INTO num_tbl VALUES + (1,'one'), + (2,'two'), + (3,'three'), + (4,'four'), + (5,'five'), + (6,'six'); +--- Users can access only their own rows +CREATE POLICY rls_tbl_policy ON rls_tbl FOR SELECT TO PUBLIC USING(owner = current_user); +ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; +GRANT ALL on rls_tbl TO PUBLIC; +GRANT ALL on num_tbl TO PUBLIC; +--- create a view owned by ivm_user +SET SESSION AUTHORIZATION ivm_user; +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls AS SELECT * FROM rls_tbl; +NOTICE: could not create an index on materialized view "ivm_rls" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + id | data | owner +----+------+---------- + 1 | foo | ivm_user +(1 row) + +RESET SESSION AUTHORIZATION; +--- inserts rows owned by different users +INSERT INTO rls_tbl VALUES + (3,'baz','ivm_user'), + (4,'qux','postgres'); +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + id | data | owner +----+------+---------- + 1 | foo | ivm_user + 3 | baz | ivm_user +(2 rows) + +--- combination of diffent kinds of commands +WITH + i AS (INSERT INTO rls_tbl VALUES(5,'quux','postgres'), (6,'corge','ivm_user')), + u AS (UPDATE rls_tbl SET owner = 'postgres' WHERE id = 1), + u2 AS (UPDATE rls_tbl SET owner = 'ivm_user' WHERE id = 2) +SELECT; +-- +(1 row) + +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + id | data | owner +----+-------+---------- + 2 | bar | ivm_user + 3 | baz | ivm_user + 6 | corge | ivm_user +(3 rows) + +--- +SET SESSION AUTHORIZATION ivm_user; +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls2 AS SELECT * FROM rls_tbl JOIN num_tbl USING(id); +NOTICE: could not create an index on materialized view "ivm_rls2" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +RESET SESSION AUTHORIZATION; +WITH + x AS (UPDATE rls_tbl SET data = data || '_2' where id in (3,4)), + y AS (UPDATE num_tbl SET num = num || '_2' where id in (3,4)) +SELECT; +-- +(1 row) + +SELECT * FROM ivm_rls2 ORDER BY 1,2,3; + id | data | owner | num +----+-------+----------+--------- + 2 | bar | ivm_user | two + 3 | baz_2 | ivm_user | three_2 + 6 | corge | ivm_user | six +(3 rows) + +-- automatic index creation +CREATE TABLE base_a (i int primary key, j int); +CREATE TABLE base_b (i int primary key, j int); +--- group by: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx1 AS SELECT i, sum(j) FROM base_a GROUP BY i; +NOTICE: created index "mv_idx1_index" on materialized view "mv_idx1" +--- distinct: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx2 AS SELECT DISTINCT j FROM base_a; +NOTICE: created index "mv_idx2_index" on materialized view "mv_idx2" +--- with all pkey columns: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx3(i_a, i_b) AS SELECT a.i, b.i FROM base_a a, base_b b; +NOTICE: created index "mv_idx3_index" on materialized view "mv_idx3" +--- missing some pkey columns: no index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx4 AS SELECT j FROM base_a; +NOTICE: could not create an index on materialized view "mv_idx4" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx5 AS SELECT a.i, b.j FROM base_a a, base_b b; +NOTICE: could not create an index on materialized view "mv_idx5" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +-- cleanup +DROP TABLE rls_tbl CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to materialized view ivm_rls +drop cascades to materialized view ivm_rls2 +DROP TABLE num_tbl CASCADE; +DROP USER ivm_user; +DROP USER ivm_admin; +DROP TABLE mv_base_b CASCADE; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to materialized view mv_ivm_1 +drop cascades to view b_view +drop cascades to materialized view b_mview +DROP TABLE mv_base_a CASCADE; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 4df9d8503b..21f7247a07 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -78,7 +78,7 @@ test: brin_bloom brin_multi # psql depends on create_am # amutils depends on geometry, create_index_spgist, hash_index, brin # ---------- -test: create_table_like alter_generic alter_operator misc async dbsize merge misc_functions sysviews tsrf tid tidscan tidrangescan collate.icu.utf8 incremental_sort create_role +test: create_table_like alter_generic alter_operator misc async dbsize merge misc_functions sysviews tsrf tid tidscan tidrangescan collate.icu.utf8 incremental_sort create_role incremental_matview # collate.*.utf8 tests cannot be run in parallel with each other test: rules psql psql_crosstab amutils stats_ext collate.linux.utf8 collate.windows.win1252 diff --git a/src/test/regress/sql/incremental_matview.sql b/src/test/regress/sql/incremental_matview.sql new file mode 100644 index 0000000000..82686f9324 --- /dev/null +++ b/src/test/regress/sql/incremental_matview.sql @@ -0,0 +1,533 @@ +-- create a table to use as a basis for views and materialized views in various combinations +CREATE TABLE mv_base_a (i int, j int); +INSERT INTO mv_base_a VALUES + (1,10), + (2,20), + (3,30), + (4,40), + (5,50); +CREATE TABLE mv_base_b (i int, k int); +INSERT INTO mv_base_b VALUES + (1,101), + (2,102), + (3,103), + (4,104); + +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_1 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) WITH NO DATA; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +REFRESH MATERIALIZED VIEW mv_ivm_1; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + +-- REFRESH WITH NO DATA +BEGIN; +CREATE FUNCTION dummy_ivm_trigger_func() RETURNS TRIGGER AS $$ + BEGIN + RETURN NULL; + END +$$ language plpgsql; + +CREATE CONSTRAINT TRIGGER dummy_ivm_trigger AFTER INSERT +ON mv_base_a FROM mv_ivm_1 FOR EACH ROW +EXECUTE PROCEDURE dummy_ivm_trigger_func(); + +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; + +REFRESH MATERIALIZED VIEW mv_ivm_1 WITH NO DATA; + +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; +ROLLBACK; + +-- immediate maintenance +BEGIN; +INSERT INTO mv_base_b VALUES(5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +UPDATE mv_base_a SET j = 0 WHERE i = 1; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +DELETE FROM mv_base_b WHERE (i,k) = (5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +ROLLBACK; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + +-- rename of IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_rename AS SELECT DISTINCT * FROM mv_base_a; +ALTER MATERIALIZED VIEW mv_ivm_rename RENAME COLUMN __ivm_count__ TO xxx; +DROP MATERIALIZED VIEW mv_ivm_rename; + +-- unique index on IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_unique AS SELECT DISTINCT * FROM mv_base_a; +CREATE UNIQUE INDEX ON mv_ivm_unique(__ivm_count__); +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__)); +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__ + 1)); +DROP MATERIALIZED VIEW mv_ivm_unique; + +-- TRUNCATE a base table in join views +BEGIN; +TRUNCATE mv_base_a; +SELECT * FROM mv_ivm_1; +ROLLBACK; + +BEGIN; +TRUNCATE mv_base_b; +SELECT * FROM mv_ivm_1; +ROLLBACK; + +-- some query syntax +BEGIN; +CREATE FUNCTION ivm_func() RETURNS int LANGUAGE 'sql' + AS 'SELECT 1' IMMUTABLE; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_func AS SELECT * FROM ivm_func(); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_no_tbl AS SELECT 1; +ROLLBACK; + +-- result of materialized view have DISTINCT clause or the duplicate result. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_duplicate AS SELECT j FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_distinct AS SELECT DISTINCT j FROM mv_base_a; +INSERT INTO mv_base_a VALUES(6,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; +SELECT * FROM mv_ivm_distinct ORDER BY 1; +DELETE FROM mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; +SELECT * FROM mv_ivm_distinct ORDER BY 1; +ROLLBACK; + +-- support SUM(), COUNT() and AVG() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(i), AVG(j) FROM mv_base_a GROUP BY i; +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +UPDATE mv_base_a SET j = 200 WHERE (i,j) = (2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +DELETE FROM mv_base_a WHERE (i,j) = (2,200); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +ROLLBACK; + +-- support COUNT(*) aggregate function +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; +ROLLBACK; + +-- TRUNCATE a base table in aggregate views +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +TRUNCATE mv_base_a; +SELECT sum, count FROM mv_ivm_agg; +SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +ROLLBACK; + +-- support aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +SELECT * FROM mv_ivm_group ORDER BY 1; +INSERT INTO mv_base_a VALUES(6,60); +SELECT * FROM mv_ivm_group ORDER BY 1; +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_group ORDER BY 1; +ROLLBACK; + +-- TRUNCATE a base table in aggregate views without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +TRUNCATE mv_base_a; +SELECT sum, count, avg FROM mv_ivm_group; +SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +ROLLBACK; + +-- resolved issue: When use AVG() function and values is indivisible, result of AVG() is incorrect. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_avg_bug AS SELECT i, SUM(j), COUNT(j), AVG(j) FROM mv_base_A GROUP BY i; +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; +INSERT INTO mv_base_a VALUES + (1,0), + (1,0), + (2,30), + (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; +DELETE FROM mv_base_a WHERE (i,j) = (1,0); +DELETE FROM mv_base_a WHERE (i,j) = (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; +ROLLBACK; + +-- support MIN(), MAX() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT i, MIN(j), MAX(j) FROM mv_base_a GROUP BY i; +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; +INSERT INTO mv_base_a VALUES + (1,11), (1,12), + (2,21), (2,22), + (3,31), (3,32), + (4,41), (4,42), + (5,51), (5,52); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; +DELETE FROM mv_base_a WHERE (i,j) IN ((1,10), (2,21), (3,32)); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; +ROLLBACK; + +-- support MIN(), MAX() aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT MIN(j), MAX(j) FROM mv_base_a; +SELECT * FROM mv_ivm_min_max; +INSERT INTO mv_base_a VALUES + (0,0), (6,60), (7,70); +SELECT * FROM mv_ivm_min_max; +DELETE FROM mv_base_a WHERE (i,j) IN ((0,0), (7,70)); +SELECT * FROM mv_ivm_min_max; +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_min_max; +ROLLBACK; + +-- Test MIN/MAX after search_path change +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min AS SELECT MIN(j) FROM mv_base_a; +SELECT * FROM mv_ivm_min; + +CREATE SCHEMA myschema; +GRANT ALL ON SCHEMA myschema TO public; +CREATE TABLE myschema.mv_base_a (j int); +INSERT INTO myschema.mv_base_a VALUES (1); + +DELETE FROM mv_base_a WHERE (i,j) = (1,10); +SELECT * FROM mv_ivm_min; + +SET search_path TO myschema,public,pg_catalog; +DELETE FROM public.mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_min; +ROLLBACK; + +-- aggregate views with column names specified +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b,c) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +ROLLBACK; + +-- support self join view and multiple change on the same table +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1, 10), (2, 20), (3, 30); +CREATE INCREMENTAL MATERIALIZED VIEW mv_self(v1, v2) AS + SELECT t1.v, t2.v FROM base_t AS t1 JOIN base_t AS t2 ON t1.i = t2.i; +SELECT * FROM mv_self ORDER BY v1; +INSERT INTO base_t VALUES (4,40); +DELETE FROM base_t WHERE i = 1; +UPDATE base_t SET v = v*10 WHERE i=2; +SELECT * FROM mv_self ORDER BY v1; +WITH + ins_t1 AS (INSERT INTO base_t VALUES (5,50) RETURNING 1), + ins_t2 AS (INSERT INTO base_t VALUES (6,60) RETURNING 1), + upd_t AS (UPDATE base_t SET v = v + 100 RETURNING 1), + dlt_t AS (DELETE FROM base_t WHERE i IN (4,5) RETURNING 1) +SELECT NULL; +SELECT * FROM mv_self ORDER BY v1; + +--- with sub-transactions +SAVEPOINT p1; +INSERT INTO base_t VALUES (7,70); +RELEASE SAVEPOINT p1; +INSERT INTO base_t VALUES (7,77); +SELECT * FROM mv_self ORDER BY v1, v2; + +ROLLBACK; + +-- support simultaneous table changes +BEGIN; +CREATE TABLE base_r (i int, v int); +CREATE TABLE base_s (i int, v int); +INSERT INTO base_r VALUES (1, 10), (2, 20), (3, 30); +INSERT INTO base_s VALUES (1, 100), (2, 200), (3, 300); +CREATE INCREMENTAL MATERIALIZED VIEW mv(v1, v2) AS + SELECT r.v, s.v FROM base_r AS r JOIN base_s AS s USING(i); +SELECT * FROM mv ORDER BY v1; +WITH + ins_r AS (INSERT INTO base_r VALUES (1,11) RETURNING 1), + ins_r2 AS (INSERT INTO base_r VALUES (3,33) RETURNING 1), + ins_s AS (INSERT INTO base_s VALUES (2,222) RETURNING 1), + upd_r AS (UPDATE base_r SET v = v + 1000 WHERE i = 2 RETURNING 1), + dlt_s AS (DELETE FROM base_s WHERE i = 3 RETURNING 1) +SELECT NULL; +SELECT * FROM mv ORDER BY v1; +ROLLBACK; + +-- support foreign reference constraints +BEGIN; +CREATE TABLE ri1 (i int PRIMARY KEY); +CREATE TABLE ri2 (i int PRIMARY KEY REFERENCES ri1(i) ON UPDATE CASCADE ON DELETE CASCADE, v int); +INSERT INTO ri1 VALUES (1),(2),(3); +INSERT INTO ri2 VALUES (1),(2),(3); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ri(i1, i2) AS + SELECT ri1.i, ri2.i FROM ri1 JOIN ri2 USING(i); +SELECT * FROM mv_ri ORDER BY i1; +UPDATE ri1 SET i=10 where i=1; +DELETE FROM ri1 WHERE i=2; +SELECT * FROM mv_ri ORDER BY i2; +ROLLBACK; + +-- views including NULL +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1,10),(2, NULL); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +SELECT * FROM mv ORDER BY i; +UPDATE base_t SET v = 20 WHERE i = 2; +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +BEGIN; +CREATE TABLE base_t (i int); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +SELECT * FROM mv ORDER BY i; +INSERT INTO base_t VALUES (1),(NULL); +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (1, 10), (1, 20); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, sum(v) FROM base_t GROUP BY i; +SELECT * FROM mv ORDER BY i; +UPDATE base_t SET v = v * 10; +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (NULL, 3), (NULL, 4), (NULL, 5); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, min(v), max(v) FROM base_t GROUP BY i; +SELECT * FROM mv ORDER BY i; +DELETE FROM base_t WHERE v = 1; +SELECT * FROM mv ORDER BY i; +DELETE FROM base_t WHERE v = 3; +SELECT * FROM mv ORDER BY i; +DELETE FROM base_t WHERE v = 5; +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +-- IMMV containing user defined type +BEGIN; + +CREATE TYPE mytype; +CREATE FUNCTION mytype_in(cstring) + RETURNS mytype AS 'int4in' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_out(mytype) + RETURNS cstring AS 'int4out' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE TYPE mytype ( + LIKE = int4, + INPUT = mytype_in, + OUTPUT = mytype_out +); + +CREATE FUNCTION mytype_eq(mytype, mytype) + RETURNS bool AS 'int4eq' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_lt(mytype, mytype) + RETURNS bool AS 'int4lt' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_cmp(mytype, mytype) + RETURNS integer AS 'btint4cmp' + LANGUAGE INTERNAL STRICT IMMUTABLE; + +CREATE OPERATOR = ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_eq); +CREATE OPERATOR < ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_lt); + +CREATE OPERATOR CLASS mytype_ops + DEFAULT FOR TYPE mytype USING btree AS + OPERATOR 1 <, + OPERATOR 3 = , + FUNCTION 1 mytype_cmp(mytype,mytype); + +CREATE TABLE t_mytype (x mytype); +CREATE INCREMENTAL MATERIALIZED VIEW mv_mytype AS + SELECT * FROM t_mytype; +INSERT INTO t_mytype VALUES ('1'::mytype); +SELECT * FROM mv_mytype; + +ROLLBACK; + +-- outer join is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv(a,b) AS SELECT a.i, b.i FROM mv_base_a a LEFT JOIN mv_base_b b ON a.i=b.i; +-- CTE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv AS + WITH b AS ( SELECT * FROM mv_base_b) SELECT a.i,a.j FROM mv_base_a a, b WHERE a.i = b.i; +-- contain system column +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm01 AS SELECT i,j,xmin FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm02 AS SELECT i,j FROM mv_base_a WHERE xmin = '610'; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT i,j,xmin::text AS x_min FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm06 AS SELECT i,j,xidsend(xmin) AS x_min FROM mv_base_a; +-- contain subquery +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm03 AS SELECT i,j FROM mv_base_a WHERE i IN (SELECT i FROM mv_base_b WHERE k < 103 ); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT * FROM mv_base_b) b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm05 AS SELECT i,j, (SELECT k FROM mv_base_b b WHERE a.i = b.i) FROM mv_base_a a; +-- contain ORDER BY +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) ORDER BY i,j,k; +-- contain HAVING +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) GROUP BY i,j,k HAVING SUM(i) > 5; + +-- contain view or materialized view +CREATE VIEW b_view AS SELECT i,k FROM mv_base_b; +CREATE MATERIALIZED VIEW b_mview AS SELECT i,k FROM mv_base_b; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT a.i,a.j FROM mv_base_a a,b_view b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT a.i,a.j FROM mv_base_a a,b_mview b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm09 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT i, COUNT(*) FROM mv_base_b GROUP BY i) b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm10 AS SELECT a.i,a.j FROM mv_base_a a WHERE EXISTS(SELECT 1 FROM mv_base_b b WHERE a.i = b.i) OR a.i > 5; + +-- contain mutable functions +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm12 AS SELECT i,j FROM mv_base_a WHERE i = random()::int; + +-- LIMIT/OFFSET is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm13 AS SELECT i,j FROM mv_base_a LIMIT 10 OFFSET 5; + +-- DISTINCT ON is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm14 AS SELECT DISTINCT ON(i) i, j FROM mv_base_a; + +-- TABLESAMPLE clause is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm15 AS SELECT i, j FROM mv_base_a TABLESAMPLE SYSTEM(50); + +-- window functions are not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm16 AS SELECT *, cume_dist() OVER (ORDER BY i) AS rank FROM mv_base_a; + +-- aggregate function with some options is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm17 AS SELECT COUNT(*) FILTER(WHERE i < 3) FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm18 AS SELECT COUNT(DISTINCT i) FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm19 AS SELECT array_agg(j ORDER BY i DESC) FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm20 AS SELECT i,SUM(j) FROM mv_base_a GROUP BY GROUPING SETS((i),()); + +-- inheritance parent is not supported +BEGIN; +CREATE TABLE parent (i int, v int); +CREATE TABLE child_a(options text) INHERITS(parent); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm21 AS SELECT * FROM parent; +ROLLBACK; + +-- UNION statement is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm22 AS SELECT i,j FROM mv_base_a UNION ALL SELECT i,k FROM mv_base_b;; + +-- empty target list is not allowed with IVM +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm25 AS SELECT FROM mv_base_a; + +-- FOR UPDATE/SHARE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm26 AS SELECT i,j FROM mv_base_a FOR UPDATE; + +-- tartget list cannot contain ivm column that start with '__ivm' +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm28 AS SELECT i AS "__ivm_count__" FROM mv_base_a; + +-- expressions specified in GROUP BY must appear in the target list. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm29 AS SELECT COUNT(i) FROM mv_base_a GROUP BY i; + +-- experssions containing an aggregate is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm30 AS SELECT sum(i)*0.5 FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm31 AS SELECT sum(i)/sum(j) FROM mv_base_a; + +-- VALUES is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_only_values1 AS values(1); + +-- views containing base tables with Row Level Security +DROP USER IF EXISTS ivm_admin; +DROP USER IF EXISTS ivm_user; +CREATE USER ivm_admin; +CREATE USER ivm_user; + +--- create a table with RLS +SET SESSION AUTHORIZATION ivm_admin; +CREATE TABLE rls_tbl(id int, data text, owner name); +INSERT INTO rls_tbl VALUES + (1,'foo','ivm_user'), + (2,'bar','postgres'); +CREATE TABLE num_tbl(id int, num text); +INSERT INTO num_tbl VALUES + (1,'one'), + (2,'two'), + (3,'three'), + (4,'four'), + (5,'five'), + (6,'six'); + +--- Users can access only their own rows +CREATE POLICY rls_tbl_policy ON rls_tbl FOR SELECT TO PUBLIC USING(owner = current_user); +ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; +GRANT ALL on rls_tbl TO PUBLIC; +GRANT ALL on num_tbl TO PUBLIC; + +--- create a view owned by ivm_user +SET SESSION AUTHORIZATION ivm_user; + +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls AS SELECT * FROM rls_tbl; +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; +RESET SESSION AUTHORIZATION; + +--- inserts rows owned by different users +INSERT INTO rls_tbl VALUES + (3,'baz','ivm_user'), + (4,'qux','postgres'); +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + +--- combination of diffent kinds of commands +WITH + i AS (INSERT INTO rls_tbl VALUES(5,'quux','postgres'), (6,'corge','ivm_user')), + u AS (UPDATE rls_tbl SET owner = 'postgres' WHERE id = 1), + u2 AS (UPDATE rls_tbl SET owner = 'ivm_user' WHERE id = 2) +SELECT; +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + +--- +SET SESSION AUTHORIZATION ivm_user; +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls2 AS SELECT * FROM rls_tbl JOIN num_tbl USING(id); +RESET SESSION AUTHORIZATION; + +WITH + x AS (UPDATE rls_tbl SET data = data || '_2' where id in (3,4)), + y AS (UPDATE num_tbl SET num = num || '_2' where id in (3,4)) +SELECT; +SELECT * FROM ivm_rls2 ORDER BY 1,2,3; + +-- automatic index creation +CREATE TABLE base_a (i int primary key, j int); +CREATE TABLE base_b (i int primary key, j int); + +--- group by: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx1 AS SELECT i, sum(j) FROM base_a GROUP BY i; + +--- distinct: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx2 AS SELECT DISTINCT j FROM base_a; + +--- with all pkey columns: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx3(i_a, i_b) AS SELECT a.i, b.i FROM base_a a, base_b b; + +--- missing some pkey columns: no index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx4 AS SELECT j FROM base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx5 AS SELECT a.i, b.j FROM base_a a, base_b b; + +-- cleanup + +DROP TABLE rls_tbl CASCADE; +DROP TABLE num_tbl CASCADE; +DROP USER ivm_user; +DROP USER ivm_admin; + +DROP TABLE mv_base_b CASCADE; +DROP TABLE mv_base_a CASCADE; -- 2.25.1 --Multipart=_Mon__28_Aug_2023_11_52_52_+0900_hj6L5h176QaSGtg7 Content-Type: text/x-diff; name="v29-0011-Add-documentations-about-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v29-0011-Add-documentations-about-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
* [PATCH v47 4/9] repack code cleanups @ 2026-03-31 14:48 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 77+ messages in thread From: Álvaro Herrera @ 2026-03-31 14:48 UTC (permalink / raw) --- src/backend/commands/cluster.c | 163 +++++++++++---------------------- 1 file changed, 51 insertions(+), 112 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fd0bfacaae6..c0220cb3380 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -2459,18 +2459,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) TupleTableSlot *spilled_tuple; TupleTableSlot *old_update_tuple; TupleTableSlot *ondisk_tuple; - MemoryContext apply_cxt; bool have_old_tuple = false; - - /* - * Use a separate memory context for the tuples and any memory needed to - * process them. - * - * XXX would this be better with GenerationContextCreate? - */ - apply_cxt = AllocSetContextCreate(TopTransactionContext, - "REPACK - apply", - ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcxt; spilled_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); @@ -2479,6 +2469,8 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) old_update_tuple = MakeSingleTupleTableSlot(RelationGetDescr(rel), &TTSOpsVirtual); + oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(chgcxt->cc_estate)); + while (true) { size_t nread; @@ -2570,14 +2562,15 @@ apply_concurrent_changes(BufFile *file, ChangeContext *chgcxt) else elog(ERROR, "unrecognized kind of change: %d", kind); - MemoryContextReset(apply_cxt); + ResetPerTupleExprContext(chgcxt->cc_estate); } /* Cleanup. */ ExecDropSingleTupleTableSlot(spilled_tuple); ExecDropSingleTupleTableSlot(ondisk_tuple); ExecDropSingleTupleTableSlot(old_update_tuple); - MemoryContextDelete(apply_cxt); + + MemoryContextSwitchTo(oldcxt); } /* @@ -2588,27 +2581,17 @@ static void apply_concurrent_insert(Relation rel, TupleTableSlot *slot, ChangeContext *chgcxt) { - List *recheck; - /* Put the tuple in the table, but make sure it won't be decoded */ table_tuple_insert(rel, slot, GetCurrentCommandId(true), HEAP_INSERT_NO_LOGICAL, NULL); - /* - * Update indexes with this new tuple. Because we're merely replaying an - * action that already happened, we have no use for the recheck list of - * indexes returned, so just free it. XXX or maybe just leave it? - */ - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - 0, - slot, - NIL, NULL); - list_free(recheck); - + /* Update indexes with this new tuple. */ + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + 0, + slot, + NIL, NULL); pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_INSERTED, 1); - - ResetPerTupleExprContext(chgcxt->cc_estate); } /* @@ -2624,10 +2607,9 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, TM_FailureData tmfd; TU_UpdateIndexes update_indexes; TM_Result res; - List *recheck; /* - * Carry out the update, avoiding logical decoding info. + * Carry out the update, skipping logical decoding for it. */ res = table_tuple_update(rel, &(ondisk_tuple->tts_tid), spilled_tuple, GetCurrentCommandId(true), @@ -2646,13 +2628,11 @@ apply_concurrent_update(Relation rel, TupleTableSlot *spilled_tuple, if (update_indexes == TU_Summarizing) flags |= EIIT_ONLY_SUMMARIZING; - recheck = ExecInsertIndexTuples(chgcxt->cc_rri, - chgcxt->cc_estate, - flags, - spilled_tuple, - NIL, NULL); - list_free(recheck); - ResetPerTupleExprContext(chgcxt->cc_estate); + ExecInsertIndexTuples(chgcxt->cc_rri, + chgcxt->cc_estate, + flags, + spilled_tuple, + NIL, NULL); } pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_UPDATED, 1); @@ -2665,10 +2645,7 @@ apply_concurrent_delete(Relation rel, TupleTableSlot *slot) TM_FailureData tmfd; /* - * Delete tuple from the new heap. - * - * Do it like in simple_heap_delete(), except for 'wal_logical' (and - * except for 'wait'). + * Delete tuple from the new heap, skipping logical decoding for it. */ res = table_tuple_delete(rel, &(slot->tts_tid), GetCurrentCommandId(true), @@ -3009,12 +2986,8 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, char relpersistence; bool is_system_catalog; Oid ident_idx_new; - XLogRecPtr wal_insert_ptr, - end_of_wal; - char dummy_rec_data = '\0'; - Relation *ind_refs, - *ind_refs_p; - int nind; + XLogRecPtr end_of_wal; + List *indexrels; ChangeContext chgcxt; /* Like in cluster_rel(). */ @@ -3041,24 +3014,23 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, */ ind_oids_new = build_new_indexes(NewHeap, OldHeap, ind_oids_old); - /* Find "identity index" on the new relation. */ + /* + * The identity index in the new relation appears in the same relative + * position as the corresponding index in the old relation. Find it. + */ ident_idx_new = InvalidOid; - forboth(lc, ind_oids_old, lc2, ind_oids_new) + foreach_oid(ind_old, ind_oids_old) { - Oid ind_old = lfirst_oid(lc); - Oid ind_new = lfirst_oid(lc2); - if (identIdx == ind_old) { - ident_idx_new = ind_new; + ident_idx_new = list_nth_oid(ind_oids_new, + foreach_current_index(ind_old)); break; } } - - /* Should not happen, given our lock on the old relation. */ if (!OidIsValid(ident_idx_new)) - ereport(ERROR, - errmsg("identity index missing on the new relation")); + elog(ERROR, "could not find index matching \"%s\" at the new relation", + get_rel_name(identIdx)); /* Gather information to apply concurrent changes. */ initialize_change_context(&chgcxt, NewHeap, ident_idx_new); @@ -3074,8 +3046,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * incomplete page; see GetInsertRecPtr), to minimize the amount of data * we need to flush while holding exclusive lock on the source table. */ - wal_insert_ptr = GetInsertRecPtr(); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3094,10 +3065,9 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, /* * Lock all indexes now, not only the clustering one: all indexes need to * have their files swapped. While doing that, store their relation - * references in an array, to handle predicate locks below. + * references in a zero-terminated array, to handle predicate locks below. */ - ind_refs_p = ind_refs = palloc_array(Relation, list_length(ind_oids_old)); - nind = 0; + indexrels = NIL; foreach_oid(ind_oid, ind_oids_old) { Relation index; @@ -3105,16 +3075,11 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, index = index_open(ind_oid, AccessExclusiveLock); /* - * TODO 1) Do we need to check if ALTER INDEX was executed since the - * new index was created in build_new_indexes()? 2) Specifically for - * the clustering index, should check_index_is_clusterable() be called - * here? (Not sure about the latter: ShareUpdateExclusiveLock on the - * table probably blocks all commands that affect the result of - * check_index_is_clusterable().) + * Some things about the index may have changed before we locked the + * index, such as ALTER INDEX RENAME. We don't need to do anything + * here to absorb those changes in the new index. */ - *ind_refs_p = index; - ind_refs_p++; - nind++; + indexrels = lappend(indexrels, index); } /* @@ -3128,40 +3093,18 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, * Tuples and pages of the old heap will be gone, but the heap will stay. */ TransferPredicateLocksToHeapRelation(OldHeap); - /* The same for indexes. */ - for (int i = 0; i < nind; i++) + foreach_ptr(RelationData, index, indexrels) { - Relation index = ind_refs[i]; - TransferPredicateLocksToHeapRelation(index); - - /* - * References to indexes on the old relation are not needed anymore, - * however locks stay till the end of the transaction. - */ index_close(index, NoLock); } - pfree(ind_refs); + list_free(indexrels); /* - * Flush anything we see in WAL, to make sure that all changes committed - * while we were waiting for the exclusive lock are available for - * decoding. This should not be necessary if all backends had - * synchronous_commit set, but we can't rely on this setting. - * - * Unfortunately, GetInsertRecPtr() may lag behind the actual insert - * position, and GetLastImportantRecPtr() points at the start of the last - * record rather than at the end. Thus the simplest way to determine the - * insert position is to insert a dummy record and use its LSN. - * - * XXX Consider using GetLastImportantRecPtr() and adding the size of the - * last record (plus the total size of all the page headers the record - * spans)? + * Flush WAL again, to make sure that all changes committed while we were + * waiting for the exclusive lock are available for decoding. */ - XLogBeginInsert(); - XLogRegisterData(&dummy_rec_data, 1); - wal_insert_ptr = XLogInsert(RM_XLOG_ID, XLOG_NOOP); - XLogFlush(wal_insert_ptr); + XLogFlush(GetXLogInsertRecPtr()); end_of_wal = GetFlushRecPtr(NULL); /* @@ -3186,10 +3129,7 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); - Oid mapped_tables[4]; - - /* Zero out possible results from swapped_relation_files */ - memset(mapped_tables, 0, sizeof(mapped_tables)); + Oid mapped_tables[4] = {0}; swap_relation_files(ind_old, ind_new, (old_table_oid == RelationRelationId), @@ -3200,13 +3140,12 @@ rebuild_relation_finish_concurrent(Relation NewHeap, Relation OldHeap, mapped_tables); #ifdef USE_ASSERT_CHECKING - /* * Concurrent processing is not supported for system relations, so * there should be no mapped tables. */ for (int i = 0; i < 4; i++) - Assert(mapped_tables[i] == 0); + Assert(!OidIsValid(mapped_tables[i])); #endif } @@ -3256,22 +3195,22 @@ build_new_indexes(Relation NewHeap, Relation OldHeap, List *OldIndexes) pgstat_progress_update_param(PROGRESS_REPACK_PHASE, PROGRESS_REPACK_PHASE_REBUILD_INDEX); - foreach_oid(ind_oid, OldIndexes) + foreach_oid(oldindex, OldIndexes) { - Oid ind_oid_new; + Oid newindex; char *newName; Relation ind; - ind = index_open(ind_oid, ShareUpdateExclusiveLock); + ind = index_open(oldindex, ShareUpdateExclusiveLock); - newName = ChooseRelationName(get_rel_name(ind_oid), + newName = ChooseRelationName(get_rel_name(oldindex), NULL, "repacknew", get_rel_namespace(ind->rd_index->indrelid), false); - ind_oid_new = index_create_copy(NewHeap, false, ind_oid, - ind->rd_rel->reltablespace, newName); - result = lappend_oid(result, ind_oid_new); + newindex = index_create_copy(NewHeap, false, oldindex, + ind->rd_rel->reltablespace, newName); + result = lappend_oid(result, newindex); index_close(ind, NoLock); } -- 2.47.3 --2fxmamo6mu2qbxgv Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v47-0005-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 77+ messages in thread
end of thread, other threads:[~2026-03-31 14:48 UTC | newest] Thread overview: 77+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-10 02:11 [PATCH v29 10/11] Add regression tests for Incremental View Maintenance Takuma Hoshiai <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]> 2026-03-31 14:48 [PATCH v47 4/9] repack code cleanups Álvaro Herrera <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox