public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 7/8] add wal_compression_method=zstd 5+ messages / 3 participants [nested] [flat]
* [PATCH 7/8] add wal_compression_method=zstd @ 2021-03-12 20:43 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Justin Pryzby @ 2021-03-12 20:43 UTC (permalink / raw) --- configure | 163 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 20 +++ src/backend/access/transam/xlogreader.c | 15 +++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 232 insertions(+) diff --git a/configure b/configure index 3afa1e87e3..c7c4c951a7 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +ZSTD_LIBS +ZSTD_CFLAGS +with_zstd LZ4_LIBS LZ4_CFLAGS with_lz4 @@ -868,6 +871,7 @@ with_libxslt with_system_tzdata with_zlib with_lz4 +with_zstd with_gnu_ld with_ssl with_openssl @@ -897,6 +901,8 @@ XML2_CFLAGS XML2_LIBS LZ4_CFLAGS LZ4_LIBS +ZSTD_CFLAGS +ZSTD_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1576,6 +1582,7 @@ Optional Packages: use system time zone data in DIR --without-zlib do not use Zlib --without-lz4 build without LZ4 support + --with-zstd build with zstd support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1605,6 +1612,8 @@ Some influential environment variables: XML2_LIBS linker flags for XML2, overriding pkg-config LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config LZ4_LIBS linker flags for LZ4, overriding pkg-config + ZSTD_CFLAGS C compiler flags for ZSTD, overriding pkg-config + ZSTD_LIBS linker flags for ZSTD, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8705,6 +8714,137 @@ fi CFLAGS="$LZ4_CFLAGS $CFLAGS" fi +# +# ZSTD +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with zstd support" >&5 +$as_echo_n "checking whether to build with zstd support... " >&6; } + + + +# Check whether --with-zstd was given. +if test "${with_zstd+set}" = set; then : + withval=$with_zstd; + case $withval in + yes) + +$as_echo "#define USE_ZSTD 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-zstd option" "$LINENO" 5 + ;; + esac + +else + with_zstd=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_zstd" >&5 +$as_echo "$with_zstd" >&6; } + + +if test "$with_zstd" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for libzstd" >&5 +$as_echo_n "checking for libzstd... " >&6; } + +if test -n "$ZSTD_CFLAGS"; then + pkg_cv_ZSTD_CFLAGS="$ZSTD_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libzstd\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libzstd") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_ZSTD_CFLAGS=`$PKG_CONFIG --cflags "libzstd" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$ZSTD_LIBS"; then + pkg_cv_ZSTD_LIBS="$ZSTD_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libzstd\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libzstd") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_ZSTD_LIBS=`$PKG_CONFIG --libs "libzstd" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + ZSTD_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libzstd" 2>&1` + else + ZSTD_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libzstd" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$ZSTD_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (libzstd) were not met: + +$ZSTD_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables ZSTD_CFLAGS +and ZSTD_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables ZSTD_CFLAGS +and ZSTD_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + ZSTD_CFLAGS=$pkg_cv_ZSTD_CFLAGS + ZSTD_LIBS=$pkg_cv_ZSTD_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$ZSTD_LIBS $LIBS" + CFLAGS="$ZSTD_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13556,6 +13696,29 @@ done CPPFLAGS=$ac_save_CPPFLAGS fi +if test "$with_zstd" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$ZSTD_CFLAGS $CPPFLAGS" + + # Verify we have zstd's header files + for ac_header in zstd.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "zstd.h" "ac_cv_header_zstd_h" "$ac_includes_default" +if test "x$ac_cv_header_zstd_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_ZSTD_H 1 +_ACEOF + +else + as_fn_error $? "zstd.h header file is required for zstd" "$LINENO" 5 +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index d62ea5f742..abac4b28f6 100644 --- a/configure.ac +++ b/configure.ac @@ -1001,6 +1001,21 @@ if test "$with_lz4" = yes; then CFLAGS="$LZ4_CFLAGS $CFLAGS" fi +# +# ZSTD +# +AC_MSG_CHECKING([whether to build with zstd support]) +PGAC_ARG_BOOL(with, zstd, no, [build with zstd support], + [AC_DEFINE([USE_ZSTD], 1, [Define to 1 to build with zstd support. (--with-zstd)])]) +AC_MSG_RESULT([$with_zstd]) +AC_SUBST(with_zstd) + +if test "$with_zstd" = yes; then + PKG_CHECK_MODULES(ZSTD, libzstd) + LIBS="$ZSTD_LIBS $LIBS" + CFLAGS="$ZSTD_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1433,6 +1448,17 @@ if test "$with_lz4" = yes; then CPPFLAGS=$ac_save_CPPFLAGS fi +if test "$with_zstd" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$ZSTD_CFLAGS $CPPFLAGS" + + # Verify we have zstd's header files + AC_CHECK_HEADERS(zstd.h, [], + [AC_MSG_ERROR([zstd.h header file is required for zstd])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f33355e44a..f2749ebf45 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -189,6 +189,9 @@ const struct config_enum_entry wal_compression_options[] = { #endif #ifdef USE_LZ4 {"lz4", WAL_COMPRESSION_LZ4, false}, +#endif +#ifdef USE_ZSTD + {"zstd", WAL_COMPRESSION_ZSTD, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 38ac5f9e94..852e5ef7af 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,6 +41,10 @@ #include "lz4.h" #endif +#ifdef USE_ZSTD +#include "zstd.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -896,6 +900,22 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, break; #endif +#ifdef USE_ZSTD + case WAL_COMPRESSION_ZSTD: + len = ZSTD_compress(dest, PGLZ_MAX_BLCKSZ, + source, orig_len, ZSTD_CLEVEL_DEFAULT); + if (ZSTD_isError(len)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("failed compressing zstd: %s", + ZSTD_getErrorName(len)))); + len = -1; + } + + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 920c47e4d7..6ce2fbfb13 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -41,6 +41,10 @@ #include "lz4.h" #endif +#ifdef USE_ZSTD +#include "zstd.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1564,6 +1568,8 @@ wal_compression_name(WalCompression compression) return "zlib"; case WAL_COMPRESSION_LZ4: return "lz4"; + case WAL_COMPRESSION_ZSTD: + return "zstd"; default: return "???"; } @@ -1619,6 +1625,15 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) break; #endif +#ifdef USE_ZSTD + case WAL_COMPRESSION_ZSTD: + decomp_result = ZSTD_decompress(tmp.data, BLCKSZ, ptr, bkpb->bimg_len); + // XXX: ZSTD_getErrorName + if (ZSTD_isError(decomp_result)) + decomp_result = -1; + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 0993de784e..30f663f448 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -329,6 +329,7 @@ typedef enum WalCompression WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, WAL_COMPRESSION_LZ4 = 2, + WAL_COMPRESSION_ZSTD = 3, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 0a6422da4f..ad26393352 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -902,6 +902,9 @@ /* Define to 1 to build with LZ4 support (--with-lz4) */ #undef USE_LZ4 +/* Define to 1 if you have the `zstd' library (-lzstd). */ +#undef USE_ZSTD + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index 14605371bb..7dfa0f31f6 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -307,6 +307,7 @@ sub GenerateFiles HAVE_LIBXML2 => undef, HAVE_LIBXSLT => undef, HAVE_LIBZ => $self->{options}->{zlib} ? 1 : undef, + HAVE_LIBZSTD => $self->{options}->{zstd} ? 1 : undef, HAVE_LINK => undef, HAVE_LOCALE_T => 1, HAVE_LONG_INT_64 => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu-- ^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: add non-option reordering to in-tree getopt_long @ 2023-06-13 03:00 Kyotaro Horiguchi <[email protected]> 0 siblings, 1 reply; 5+ messages in thread From: Kyotaro Horiguchi @ 2023-06-13 03:00 UTC (permalink / raw) To: [email protected]; +Cc: pgsql-hackers At Fri, 9 Jun 2023 16:22:57 -0700, Nathan Bossart <[email protected]> wrote in > While working on 2dcd157, I noticed cfbot failures for Windows due to tests > with commands that had non-options specified before options. For example, > "createuser myrole -a myadmin" specified a non-option (myrole) before the > option/argument pair (-a admin). To get the tests passing for Windows, > non-options must be placed at the end (e.g., "createuser -a myadmin > myrole"). This same problem was encountered while working on 08951a7 [0], > but it was again fortunately caught with cfbot. Others have not been so > lucky [1] [2] [3]. While I don't see it as reason to change the behavior, I do believe the change could be beneficial from a user's perspective. > The reason for this discrepancy is because Windows uses the in-tree > implementation of getopt_long(), which differs from the other > implementations I've found in that it doesn't reorder non-options to the > end of argv by default. Instead, it returns -1 as soon as the first > non-option is found, even if there are other options listed afterwards. By > moving non-options to the end of argv, getopt_long() can parse all > specified options and return -1 when only non-options remain. The > implementations I reviewed all reorder argv unless the POSIXLY_CORRECT > environment variable is set or the "optstring" argument begins with '+'. > > The best reasons I can think of to keep the current behavior are 1) > reordering involves writing to the original argv array, which could be > risky (as noted by Tom [4]) and 2) any systems with a getopt_long() > implementation that doesn't reorder non-options could begin failing tests > that take advantage of this behavior. However, this quirk in the in-tree > getopt_long() is periodically missed by hackers, the only systems I'm aware > of that use it are Windows and AIX, and every other implementation of > getopt_long() I saw reorders non-options by default. Furthermore, C99 > omits const decorations in main()'s signature, so modifying argv might not > be too scary. POSIXLY_CORRECT appears to be intended for debugging or feature validation. If we know we can always rearrange argv on those platforms, we don't need it. I would suggest that we turn on the new feature at the compile time on those platforms where we know this rearrangement works, instead of switching at runtime. > Thus, I propose introducing this non-option reordering behavior but > allowing it to be disabled via the POSIXLY_CORRECT environment variable. > The attached patch is my first attempt at implementing this proposal. I > don't think we need to bother with handling '+' at the beginning of > "optstring" since it seems unlikely to be used in PostgreSQL, but it would > probably be easy enough to add if folks want it. > > I briefly looked at getopt() and concluded that we should probably retain > its POSIX-compliant behavior for non-options, as reordering support seems > much less universal than with getopt_long(). AFAICT all client utilities > use getopt_long(), anyway. > > Thoughts? As far as I can see, getopt_long on Rocky9 does *not* rearrange argv until it reaches the end of the array. But it won't matter much. regards. -- Kyotaro Horiguchi NTT Open Source Software Center ^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: add non-option reordering to in-tree getopt_long @ 2023-06-13 05:13 Nathan Bossart <[email protected]> parent: Kyotaro Horiguchi <[email protected]> 0 siblings, 1 reply; 5+ messages in thread From: Nathan Bossart @ 2023-06-13 05:13 UTC (permalink / raw) To: Kyotaro Horiguchi <[email protected]>; +Cc: pgsql-hackers On Tue, Jun 13, 2023 at 12:00:01PM +0900, Kyotaro Horiguchi wrote: > POSIXLY_CORRECT appears to be intended for debugging or feature > validation. If we know we can always rearrange argv on those > platforms, we don't need it. I would suggest that we turn on the new > feature at the compile time on those platforms where we know this > rearrangement works, instead of switching at runtime. I'd be okay with leaving it out wherever possible. I'm curious whether any supported systems do not allow this. > As far as I can see, getopt_long on Rocky9 does *not* rearrange argv > until it reaches the end of the array. But it won't matter much. Do you mean that it rearranges argv once all the options have been returned, or that it doesn't rearrange argv at all? -- Nathan Bossart Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: add non-option reordering to in-tree getopt_long @ 2023-06-13 07:02 Kyotaro Horiguchi <[email protected]> parent: Nathan Bossart <[email protected]> 0 siblings, 1 reply; 5+ messages in thread From: Kyotaro Horiguchi @ 2023-06-13 07:02 UTC (permalink / raw) To: [email protected]; +Cc: pgsql-hackers At Mon, 12 Jun 2023 22:13:43 -0700, Nathan Bossart <[email protected]> wrote in > On Tue, Jun 13, 2023 at 12:00:01PM +0900, Kyotaro Horiguchi wrote: > > POSIXLY_CORRECT appears to be intended for debugging or feature > > validation. If we know we can always rearrange argv on those > > platforms, we don't need it. I would suggest that we turn on the new > > feature at the compile time on those platforms where we know this > > rearrangement works, instead of switching at runtime. > > I'd be okay with leaving it out wherever possible. I'm curious whether any > supported systems do not allow this. Hmm. from the initial mail, I got the impression that AIX and Windows allow this, so I thought that we can do that for them. While there could be other platforms that allow it, perhaps we don't need to go as far as extending this until we come across another platform that does. > > As far as I can see, getopt_long on Rocky9 does *not* rearrange argv > > until it reaches the end of the array. But it won't matter much. > > Do you mean that it rearranges argv once all the options have been > returned, or that it doesn't rearrange argv at all? I meant the former. argv remains unaltered until getopt_long returns -1. Once it does, non-optional arguments are being collected at the end of argv. (But in my opinion, that behavior isn't very significant in this context..) regards. -- Kyotaro Horiguchi NTT Open Source Software Center ^ permalink raw reply [nested|flat] 5+ messages in thread
* Re: add non-option reordering to in-tree getopt_long @ 2023-06-13 22:36 Nathan Bossart <[email protected]> parent: Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 5+ messages in thread From: Nathan Bossart @ 2023-06-13 22:36 UTC (permalink / raw) To: Kyotaro Horiguchi <[email protected]>; +Cc: pgsql-hackers On Tue, Jun 13, 2023 at 04:02:01PM +0900, Kyotaro Horiguchi wrote: > Hmm. from the initial mail, I got the impression that AIX and Windows > allow this, so I thought that we can do that for them. While there > could be other platforms that allow it, perhaps we don't need to go as > far as extending this until we come across another platform that does. Windows seems to allow rearranging argv, based upon cfbot's results. I do not know about AIX. In any case, C99 explicitly mentions that argv should be modifiable. >> > As far as I can see, getopt_long on Rocky9 does *not* rearrange argv >> > until it reaches the end of the array. But it won't matter much. >> >> Do you mean that it rearranges argv once all the options have been >> returned, or that it doesn't rearrange argv at all? > > I meant the former. argv remains unaltered until getopt_long returns > -1. Once it does, non-optional arguments are being collected at the > end of argv. (But in my opinion, that behavior isn't very significant > in this context..) Got it. -- Nathan Bossart Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 5+ messages in thread
end of thread, other threads:[~2023-06-13 22:36 UTC | newest] Thread overview: 5+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-12 20:43 [PATCH 7/8] add wal_compression_method=zstd Justin Pryzby <[email protected]> 2023-06-13 03:00 Re: add non-option reordering to in-tree getopt_long Kyotaro Horiguchi <[email protected]> 2023-06-13 05:13 ` Re: add non-option reordering to in-tree getopt_long Nathan Bossart <[email protected]> 2023-06-13 07:02 ` Re: add non-option reordering to in-tree getopt_long Kyotaro Horiguchi <[email protected]> 2023-06-13 22:36 ` Re: add non-option reordering to in-tree getopt_long Nathan Bossart <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox