agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH 3/6] Default to zstd.. 3+ messages / 2 participants [nested] [flat]
* [PATCH 3/6] Default to zstd.. @ 2021-03-12 21:35 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 3+ messages in thread From: Justin Pryzby @ 2021-03-12 21:35 UTC (permalink / raw) for CI, not for merge --- .cirrus.yml | 14 ++++++++++++++ configure | 6 ++++-- configure.ac | 2 +- src/backend/access/transam/xlog.c | 2 +- src/backend/utils/misc/guc.c | 2 +- src/tools/ci/windows_build_config.pl | 1 + 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index d10b0a82f9..4735c67608 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -77,6 +77,7 @@ task: mkdir -m 770 /tmp/cores chown root:postgres /tmp/cores sysctl kern.corefile='/tmp/cores/%N.%P.core' + pkg install -y zstd # NB: Intentionally build without --with-llvm. The freebsd image size is # already large enough to make VM startup slow, and even without llvm @@ -184,6 +185,8 @@ task: mkdir -m 770 /tmp/cores chown root:postgres /tmp/cores sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core' + apt-get update + apt-get -y install libzstd-dev configure_script: | su postgres <<-EOF @@ -265,6 +268,7 @@ task: openldap \ openssl \ python \ + zstd \ tcl-tk brew cleanup -s # to reduce cache size @@ -385,6 +389,12 @@ task: set configure_script: + # XXX: not working + - choco install -y cmake + - curl -L -o zstd.zip https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.zip + - unzip zstd.zip + - cd zstd-1.5.2 && vcvarsall x64 && build\VS_scripts\build.VS2017.cmd && cd .. + #- cp zstd-1.5.2/build/VS_scripts/build.VS2017.cmd # copy errors out when using forward slashes - copy src\tools\ci\windows_build_config.pl src\tools\msvc\config.pl - vcvarsall x64 @@ -467,6 +477,10 @@ task: image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest cpu: $CPUS + setup_os_script: | + apt-get update + apt-get -y install libzstd-dev + sysinfo_script: | id uname -a diff --git a/configure b/configure index f3cb5c2b51..b88917fcb5 100755 --- a/configure +++ b/configure @@ -1584,7 +1584,7 @@ Optional Packages: use system time zone data in DIR --without-zlib do not use Zlib --with-lz4 build with LZ4 support - --with-zstd build with ZSTD support + --without-zstd build without Zstd support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -9070,7 +9070,9 @@ $as_echo "#define USE_ZSTD 1" >>confdefs.h esac else - with_zstd=no + with_zstd=yes + +$as_echo "#define USE_ZSTD 1" >>confdefs.h fi diff --git a/configure.ac b/configure.ac index 19d1a80367..4ae681d8bf 100644 --- a/configure.ac +++ b/configure.ac @@ -1060,7 +1060,7 @@ fi # ZSTD # AC_MSG_CHECKING([whether to build with ZSTD support]) -PGAC_ARG_BOOL(with, zstd, no, [build with ZSTD support], +PGAC_ARG_BOOL(with, zstd, yes, [build with ZSTD support], [AC_DEFINE([USE_ZSTD], 1, [Define to 1 to build with ZSTD support. (--with-zstd)])]) AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 0d2bd7a357..9aa4d3eaea 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -120,7 +120,7 @@ char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; -int wal_compression = WAL_COMPRESSION_NONE; +int wal_compression = WAL_COMPRESSION_ZSTD; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 66e6d664b0..96854b6a2b 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4900,7 +4900,7 @@ static struct config_enum ConfigureNamesEnum[] = NULL }, &wal_compression, - WAL_COMPRESSION_NONE, wal_compression_options, + WAL_COMPRESSION_ZSTD, wal_compression_options, NULL, NULL, NULL }, diff --git a/src/tools/ci/windows_build_config.pl b/src/tools/ci/windows_build_config.pl index b0d4360c74..97ce79ea7f 100644 --- a/src/tools/ci/windows_build_config.pl +++ b/src/tools/ci/windows_build_config.pl @@ -9,5 +9,6 @@ $config->{"asserts"} = 1; $config->{"openssl"} = "c:/openssl/1.1/"; $config->{"perl"} = "c:/strawberry/$ENV{DEFAULT_PERL_VERSION}/perl/"; $config->{"python"} = "c:/python/"; +$config->{"zstd"} = "c:/zstd/"; 1; -- 2.17.1 --pAwQNkOnpTn9IO2O Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-Use-GUC-hooks-to-support-compression-level.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH 08/10] Default to zstd.. @ 2021-03-12 21:35 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 3+ messages in thread From: Justin Pryzby @ 2021-03-12 21:35 UTC (permalink / raw) for CI, not for merge --- configure | 6 ++++-- configure.ac | 2 +- src/backend/access/transam/xlog.c | 2 +- src/backend/utils/misc/guc.c | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/configure b/configure index 81e23418b2..253f028fc4 100755 --- a/configure +++ b/configure @@ -1582,7 +1582,7 @@ Optional Packages: use system time zone data in DIR --without-zlib do not use Zlib --without-lz4 build without LZ4 support - --with-zstd build with Zstd compression library + --without-zstd build without Zstd compression library --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -8740,7 +8740,9 @@ $as_echo "#define USE_ZSTD 1" >>confdefs.h esac else - with_zstd=no + with_zstd=yes + +$as_echo "#define USE_ZSTD 1" >>confdefs.h fi diff --git a/configure.ac b/configure.ac index d6f6349067..8d72710fa7 100644 --- a/configure.ac +++ b/configure.ac @@ -1005,7 +1005,7 @@ fi # ZSTD # AC_MSG_CHECKING([whether to build with zstd support]) -PGAC_ARG_BOOL(with, zstd, no, [build with Zstd compression library], +PGAC_ARG_BOOL(with, zstd, yes, [build without Zstd compression library], [AC_DEFINE([USE_ZSTD], 1, [Define to 1 to build with zstd support. (--with-zstd)])]) AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 307eee6626..92023de9f5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_LZ4; +int wal_compression_method = WAL_COMPRESSION_ZSTD; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 52f9cd0242..8031e027aa 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4728,7 +4728,7 @@ static struct config_enum ConfigureNamesEnum[] = NULL }, &wal_compression_method, - WAL_COMPRESSION_LZ4, wal_compression_options, + WAL_COMPRESSION_ZSTD, wal_compression_options, NULL, NULL, NULL }, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0009-Add-zstd-compression-levels.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH] Reproduce filtering issue. @ 2026-03-23 11:50 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 3+ messages in thread From: Antonin Houska @ 2026-03-23 11:50 UTC (permalink / raw) --- contrib/test_decoding/expected/filtering.out | 74 ++++++++++++++ contrib/test_decoding/specs/filtering.spec | 101 +++++++++++++++++++ src/backend/executor/nodeModifyTable.c | 2 + src/backend/replication/logical/snapbuild.c | 3 + src/test/isolation/isolationtester.c | 9 +- 5 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 contrib/test_decoding/expected/filtering.out create mode 100644 contrib/test_decoding/specs/filtering.spec diff --git a/contrib/test_decoding/expected/filtering.out b/contrib/test_decoding/expected/filtering.out new file mode 100644 index 00000000000..6ba9690509f --- /dev/null +++ b/contrib/test_decoding/expected/filtering.out @@ -0,0 +1,74 @@ +Parsed test spec with 5 sessions + +starting permutation: s1_assign_xid s3_repack s2_assign_xid s1_rollback s4_insert s5_wakeup_snapbuild s2_rollback s5_wakeup_insert_speculative s4_insert_commit s5_wakeup_repack +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s1_assign_xid: + BEGIN; + CREATE TABLE c(i int); + +step s3_repack: + REPACK (CONCURRENTLY) a; + <waiting ...> +step s2_assign_xid: + BEGIN; + CREATE TABLE d(i int); + +step s1_rollback: + ROLLBACK; + +step s4_insert: + BEGIN; + INSERT INTO t(i) + SELECT max(i) + 1 FROM t ON CONFLICT (i) DO UPDATE SET i=EXCLUDED.i; + <waiting ...> +step s5_wakeup_snapbuild: + SELECT injection_points_wakeup('snapbuild-full'); + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_rollback: + ROLLBACK; + +step s5_wakeup_insert_speculative: + SELECT injection_points_wakeup('insert-speculative-before-confirm'); + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_insert: <... completed> +step s4_insert_commit: + COMMIT; + +step s5_wakeup_repack: + SELECT injection_points_wakeup('repack-concurrently-before-lock'); + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_repack: <... completed> +injection_points_detach +----------------------- + +(1 row) + +injection_points_detach +----------------------- + +(1 row) + diff --git a/contrib/test_decoding/specs/filtering.spec b/contrib/test_decoding/specs/filtering.spec new file mode 100644 index 00000000000..a02c9f8facb --- /dev/null +++ b/contrib/test_decoding/specs/filtering.spec @@ -0,0 +1,101 @@ +setup +{ + CREATE TABLE a(i int primary key, j int) WITH (autovacuum_enabled = off); + INSERT INTO a(i, j) VALUES (1, 1), (2, 2); + CREATE TABLE t(i int primary key); + INSERT INTO t(i) VALUES (1); + CREATE EXTENSION injection_points; +} + +session s1 +step s1_assign_xid +{ + BEGIN; + CREATE TABLE c(i int); +} +step s1_rollback +{ + ROLLBACK; +} + +session s2 +step s2_assign_xid +{ + BEGIN; + CREATE TABLE d(i int); +} +step s2_rollback +{ + ROLLBACK; +} + +session s3 +setup +{ + SELECT injection_points_attach('snapbuild-full', 'wait'); + SELECT injection_points_attach('repack-concurrently-before-lock', 'wait'); +} +step s3_repack +{ + REPACK (CONCURRENTLY) a; +} +teardown +{ + SELECT injection_points_detach('repack-concurrently-before-lock'); + SELECT injection_points_detach('snapbuild-full'); +} + +session s4 +setup +{ + SELECT injection_points_set_local(); + SELECT injection_points_attach('insert-speculative-before-confirm', 'wait'); +} +step s4_insert +{ + BEGIN; + INSERT INTO t(i) + SELECT max(i) + 1 FROM t ON CONFLICT (i) DO UPDATE SET i=EXCLUDED.i; +} +step s4_insert_commit +{ + COMMIT; +} +teardown +{ + SELECT injection_points_detach('insert-speculative-before-confirm'); +} + +session s5 +step s5_wakeup_snapbuild +{ + SELECT injection_points_wakeup('snapbuild-full'); +} +step s5_wakeup_insert_speculative +{ + SELECT injection_points_wakeup('insert-speculative-before-confirm'); +} +step s5_wakeup_repack +{ + SELECT injection_points_wakeup('repack-concurrently-before-lock'); +} + +permutation +# Bring the snapshot builder to the FULL_SNAPSHOT state. +s1_assign_xid +s3_repack +s2_assign_xid +s1_rollback +# Perform the speculative insert, but no confirmation so far. The snapshot +# builder should decode it. +s4_insert +# Let the snapshout builder achieve CONSISTENT state and finish the setup. +s5_wakeup_snapbuild +s2_rollback +# While REPACK is waiting on repack-concurrently-before-lock, let the insert +# get confirmed. Relation filtering is now enabled. +s5_wakeup_insert_speculative +s4_insert_commit +# REPACK should now decode the speculative insert and decode the speculative +# insert (with the confirmation record filtered out). +s5_wakeup_repack diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 680c29f35d5..6d5482e5746 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -1232,6 +1232,8 @@ ExecInsert(ModifyTableContext *context, slot, arbiterIndexes, &specConflict); + INJECTION_POINT("insert-speculative-before-confirm", NULL); + /* adjust the tuple's state accordingly */ table_tuple_complete_speculative(resultRelationDesc, slot, specToken, !specConflict); diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index fbdd4600a2b..883e5b6e18f 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -141,6 +141,7 @@ #include "storage/procarray.h" #include "storage/standby.h" #include "utils/builtins.h" +#include "utils/injection_point.h" #include "utils/memutils.h" #include "utils/snapmgr.h" #include "utils/snapshot.h" @@ -1390,6 +1391,8 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn builder->state = SNAPBUILD_FULL_SNAPSHOT; builder->next_phase_at = running->nextXid; + INJECTION_POINT("snapbuild-full", NULL); + ereport(LOG, errmsg("logical decoding found initial consistent point at %X/%08X", LSN_FORMAT_ARGS(lsn)), diff --git a/src/test/isolation/isolationtester.c b/src/test/isolation/isolationtester.c index 440c875b8ac..8f17ee412c9 100644 --- a/src/test/isolation/isolationtester.c +++ b/src/test/isolation/isolationtester.c @@ -216,15 +216,22 @@ main(int argc, char **argv) * exactly expect concurrent use of test tables. However, autovacuum will * occasionally take AccessExclusiveLock to truncate a table, and we must * ignore that transient wait. + * + * If the session's backend is blocked, and if its background worker is + * waiting on an injection point, we assume that the injection point is + * the reason for the backend to be blocked. That's what we check in the + * second query of the UNION. XXX Should we use a separate query for that? */ initPQExpBuffer(&wait_query); appendPQExpBufferStr(&wait_query, + "WITH blocking(res) AS (" "SELECT pg_catalog.pg_isolation_test_session_is_blocked($1, '{"); /* The spec syntax requires at least one session; assume that here. */ appendPQExpBufferStr(&wait_query, conns[1].backend_pid_str); for (i = 2; i < nconns; i++) appendPQExpBuffer(&wait_query, ",%s", conns[i].backend_pid_str); - appendPQExpBufferStr(&wait_query, "}')"); + appendPQExpBufferStr(&wait_query, "}') UNION " + "SELECT pg_catalog.pg_isolation_test_session_is_blocked(pid, '{}') FROM pg_stat_activity WHERE leader_pid=$1) SELECT bool_or(res) FROM blocking"); res = PQprepare(conns[0].conn, PREP_WAITING, wait_query.data, 0, NULL); if (PQresultStatus(res) != PGRES_COMMAND_OK) -- 2.47.3 --=-=-=-- ^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2026-03-23 11:50 UTC | newest] Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-12 21:35 [PATCH 3/6] Default to zstd.. Justin Pryzby <pryzbyj@telsasoft.com> 2021-03-12 21:35 [PATCH 08/10] Default to zstd.. Justin Pryzby <pryzbyj@telsasoft.com> 2026-03-23 11:50 [PATCH] Reproduce filtering issue. Antonin Houska <ah@cybertec.at>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox