public inbox for [email protected]
help / color / mirror / Atom feedFrom: Mihail Nikalayeu <[email protected]>
To: Alvaro Herrera <[email protected]>
Cc: Pg Hackers <[email protected]>
Cc: Antonin Houska <[email protected]>
Cc: Robert Treat <[email protected]>
Cc: Fujii Masao <[email protected]>
Subject: Re: Adding REPACK [concurrently]
Date: Sun, 31 Aug 2025 17:29:00 +0200
Message-ID: <CADzfLwWJqoG6uPt+HywKOFjXhqSbfCr+VXpfio9YQ6yqQaihPA@mail.gmail.com> (raw)
In-Reply-To: <[email protected]>
References: <[email protected]>
<[email protected]>
Hello!
I started an attempt to make a "lightweight" MVCC-safe prototype and
stuck into the "it is not working" issue.
After some debugging I realized Antonin's variant (catalog-mode based)
seems to be broken also...
And after a few more hours I realized non-MVCC is broken as well :)
This is a patch with a test to reproduce the issue related to repack +
concurrent modifications.
Seems like some updates may be lost.
I hope the patch logic is clear - but feel free to ask if not.
Best regards,
Mikhail.
Attachments:
[application/octet-stream] v22-0002-Add-stress-tests-for-concurrent-index-builds.patch (9.1K, 2-v22-0002-Add-stress-tests-for-concurrent-index-builds.patch)
download | inline diff:
From c7424f44a086433d2eff6153476e0fd0c6b5b576 Mon Sep 17 00:00:00 2001
From: Mikhail Nikalayeu <[email protected]>
Date: Sat, 30 Nov 2024 16:24:20 +0100
Subject: [PATCH v22 02/12] Add stress tests for concurrent index builds
Introduce stress tests for concurrent index operations:
- test concurrent inserts/updates during CREATE/REINDEX INDEX CONCURRENTLY
- cover various index types (btree, gin, gist, brin, hash, spgist)
- test unique and non-unique indexes
- test with expressions and predicates
- test both parallel and non-parallel operations
These tests verify the behavior of the following commits.
---
src/bin/pg_amcheck/meson.build | 1 +
src/bin/pg_amcheck/t/006_cic.pl | 223 ++++++++++++++++++++++++++++++++
2 files changed, 224 insertions(+)
create mode 100644 src/bin/pg_amcheck/t/006_cic.pl
diff --git a/src/bin/pg_amcheck/meson.build b/src/bin/pg_amcheck/meson.build
index 316ea0d40b8..7df15435fbb 100644
--- a/src/bin/pg_amcheck/meson.build
+++ b/src/bin/pg_amcheck/meson.build
@@ -28,6 +28,7 @@ tests += {
't/003_check.pl',
't/004_verify_heapam.pl',
't/005_opclass_damage.pl',
+ 't/006_cic.pl',
],
},
}
diff --git a/src/bin/pg_amcheck/t/006_cic.pl b/src/bin/pg_amcheck/t/006_cic.pl
new file mode 100644
index 00000000000..2aad0e8daa8
--- /dev/null
+++ b/src/bin/pg_amcheck/t/006_cic.pl
@@ -0,0 +1,223 @@
+# Copyright (c) 2024, PostgreSQL Global Development Group
+
+# Test REINDEX CONCURRENTLY with concurrent modifications and HOT updates
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+Test::More->builder->todo_start('filesystem bug')
+ if PostgreSQL::Test::Utils::has_wal_read_bug;
+
+my ($node, $result);
+
+#
+# Test set-up
+#
+$node = PostgreSQL::Test::Cluster->new('RC_test');
+$node->init;
+$node->append_conf('postgresql.conf',
+ 'lock_timeout = ' . (1000 * $PostgreSQL::Test::Utils::timeout_default));
+$node->append_conf('postgresql.conf', 'fsync = off');
+$node->start;
+$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
+$node->safe_psql('postgres', q(CREATE TABLE tbl(i int primary key,
+ c1 money default 0, c2 money default 0,
+ c3 money default 0, updated_at timestamp,
+ ia int4[], p point)));
+$node->safe_psql('postgres', q(CREATE INDEX CONCURRENTLY idx ON tbl(i, updated_at);));
+# create sequence
+$node->safe_psql('postgres', q(CREATE UNLOGGED SEQUENCE in_row_rebuild START 1 INCREMENT 1;));
+$node->safe_psql('postgres', q(SELECT nextval('in_row_rebuild');));
+
+# Create helper functions for predicate tests
+$node->safe_psql('postgres', q(
+ CREATE FUNCTION predicate_stable() RETURNS bool IMMUTABLE
+ LANGUAGE plpgsql AS $$
+ BEGIN
+ EXECUTE 'SELECT txid_current()';
+ RETURN true;
+ END; $$;
+));
+
+$node->safe_psql('postgres', q(
+ CREATE FUNCTION predicate_const(integer) RETURNS bool IMMUTABLE
+ LANGUAGE plpgsql AS $$
+ BEGIN
+ RETURN MOD($1, 2) = 0;
+ END; $$;
+));
+
+# Run CIC/RIC in different options concurrently with upserts
+$node->pgbench(
+ '--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
+ 0,
+ [qr{actually processed}],
+ [qr{^$}],
+ 'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY',
+ {
+ 'concurrent_ops' => q(
+ SET debug_parallel_query = off; -- this is because predicate_stable implementation
+ SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
+ \if :gotlock
+ SELECT nextval('in_row_rebuild') AS last_value \gset
+ \set variant random(0, 5)
+ \set parallels random(0, 4)
+ \if :last_value < 3
+ ALTER TABLE tbl SET (parallel_workers=:parallels);
+ \if :variant = 0
+ CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at);
+ \elif :variant = 1
+ CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at) WHERE predicate_stable();
+ \elif :variant = 2
+ CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at) WHERE MOD(i, 2) = 0;
+ \elif :variant = 3
+ CREATE INDEX CONCURRENTLY new_idx ON tbl(i, updated_at) WHERE predicate_const(i);
+ \elif :variant = 4
+ CREATE INDEX CONCURRENTLY new_idx ON tbl(predicate_const(i));
+ \elif :variant = 5
+ CREATE INDEX CONCURRENTLY new_idx ON tbl(i, predicate_const(i), updated_at) WHERE predicate_const(i);
+ \endif
+ \sleep 10 ms
+ SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
+ REINDEX INDEX CONCURRENTLY new_idx;
+ \sleep 10 ms
+ SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
+ DROP INDEX CONCURRENTLY new_idx;
+ \endif
+ SELECT pg_advisory_unlock(42);
+ \else
+ \set num random(1000, 100000)
+ BEGIN;
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ SELECT setval('in_row_rebuild', 1);
+ COMMIT;
+ \endif
+ )
+ });
+
+$node->safe_psql('postgres', q(TRUNCATE TABLE tbl;));
+
+# Run CIC/RIC for unique index concurrently with upserts
+$node->pgbench(
+ '--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
+ 0,
+ [qr{actually processed}],
+ [qr{^$}],
+ 'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY for unique BTREE',
+ {
+ 'concurrent_ops_unique_idx' => q(
+ SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
+ \if :gotlock
+ SELECT nextval('in_row_rebuild') AS last_value \gset
+ \set parallels random(0, 4)
+ \if :last_value < 3
+ ALTER TABLE tbl SET (parallel_workers=:parallels);
+ CREATE UNIQUE INDEX CONCURRENTLY new_idx ON tbl(i);
+ \sleep 10 ms
+ SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
+ REINDEX INDEX CONCURRENTLY new_idx;
+ \sleep 10 ms
+ SELECT bt_index_check('new_idx', heapallindexed => true, checkunique => true);
+ DROP INDEX CONCURRENTLY new_idx;
+ \endif
+ SELECT pg_advisory_unlock(42);
+ \else
+ \set num random(1, power(10, random(1, 5)))
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ SELECT setval('in_row_rebuild', 1);
+ \endif
+ )
+ });
+
+$node->safe_psql('postgres', q(TRUNCATE TABLE tbl;));
+
+# Run CIC/RIC for GIN with upserts
+$node->pgbench(
+ '--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
+ 0,
+ [qr{actually processed}],
+ [qr{^$}],
+ 'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY for GIN/GIST/BRIN/HASH/SPGIST',
+ {
+ 'concurrent_ops_gin_idx' => q(
+ SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
+ \if :gotlock
+ SELECT nextval('in_row_rebuild') AS last_value \gset
+ \set parallels random(0, 4)
+ \if :last_value < 3
+ ALTER TABLE tbl SET (parallel_workers=:parallels);
+ CREATE INDEX CONCURRENTLY new_idx ON tbl USING GIN (ia);
+ \sleep 10 ms
+ SELECT gin_index_check('new_idx');
+ REINDEX INDEX CONCURRENTLY new_idx;
+ \sleep 10 ms
+ SELECT gin_index_check('new_idx');
+ DROP INDEX CONCURRENTLY new_idx;
+ \endif
+ SELECT pg_advisory_unlock(42);
+ \else
+ \set num random(1, power(10, random(1, 5)))
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ SELECT setval('in_row_rebuild', 1);
+ \endif
+ )
+ });
+
+$node->safe_psql('postgres', q(TRUNCATE TABLE tbl;));
+
+# Run CIC/RIC for GIST/BRIN/HASH/SPGIST index concurrently with upserts
+$node->pgbench(
+ '--no-vacuum --client=30 --jobs=4 --exit-on-abort --transactions=2500',
+ 0,
+ [qr{actually processed}],
+ [qr{^$}],
+ 'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY for GIN/GIST/BRIN/HASH/SPGIST',
+ {
+ 'concurrent_ops_other_idx' => q(
+ SELECT pg_try_advisory_lock(42)::integer AS gotlock \gset
+ \if :gotlock
+ SELECT nextval('in_row_rebuild') AS last_value \gset
+ \set parallels random(0, 4)
+ \if :last_value < 3
+ ALTER TABLE tbl SET (parallel_workers=:parallels);
+ \set variant random(0, 3)
+ \if :variant = 0
+ CREATE INDEX CONCURRENTLY new_idx ON tbl USING GIST (p);
+ \elif :variant = 1
+ CREATE INDEX CONCURRENTLY new_idx ON tbl USING BRIN (updated_at);
+ \elif :variant = 2
+ CREATE INDEX CONCURRENTLY new_idx ON tbl USING HASH (updated_at);
+ \elif :variant = 3
+ CREATE INDEX CONCURRENTLY new_idx ON tbl USING SPGIST (p);
+ \endif
+ \sleep 10 ms
+ REINDEX INDEX CONCURRENTLY new_idx;
+ \sleep 10 ms
+ DROP INDEX CONCURRENTLY new_idx;
+ \endif
+ SELECT pg_advisory_unlock(42);
+ \else
+ \set num random(1, power(10, random(1, 5)))
+ INSERT INTO tbl VALUES(floor(random()*:num),0,0,0,now())
+ ON CONFLICT(i) DO UPDATE SET updated_at = now();
+ SELECT setval('in_row_rebuild', 1);
+ \endif
+ )
+ });
+
+$node->stop;
+done_testing();
\ No newline at end of file
--
2.43.0
view thread (106+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected], [email protected], [email protected], [email protected], [email protected], [email protected]
Subject: Re: Adding REPACK [concurrently]
In-Reply-To: <CADzfLwWJqoG6uPt+HywKOFjXhqSbfCr+VXpfio9YQ6yqQaihPA@mail.gmail.com>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox