public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 7/8] add wal_compression_method=zstd 3+ messages / 3 participants [nested] [flat]
* [PATCH 7/8] add wal_compression_method=zstd @ 2021-03-12 20:43 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Justin Pryzby @ 2021-03-12 20:43 UTC (permalink / raw) --- configure | 163 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 20 +++ src/backend/access/transam/xlogreader.c | 15 +++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 232 insertions(+) diff --git a/configure b/configure index 3afa1e87e3..c7c4c951a7 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +ZSTD_LIBS +ZSTD_CFLAGS +with_zstd LZ4_LIBS LZ4_CFLAGS with_lz4 @@ -868,6 +871,7 @@ with_libxslt with_system_tzdata with_zlib with_lz4 +with_zstd with_gnu_ld with_ssl with_openssl @@ -897,6 +901,8 @@ XML2_CFLAGS XML2_LIBS LZ4_CFLAGS LZ4_LIBS +ZSTD_CFLAGS +ZSTD_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1576,6 +1582,7 @@ Optional Packages: use system time zone data in DIR --without-zlib do not use Zlib --without-lz4 build without LZ4 support + --with-zstd build with zstd support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1605,6 +1612,8 @@ Some influential environment variables: XML2_LIBS linker flags for XML2, overriding pkg-config LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config LZ4_LIBS linker flags for LZ4, overriding pkg-config + ZSTD_CFLAGS C compiler flags for ZSTD, overriding pkg-config + ZSTD_LIBS linker flags for ZSTD, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8705,6 +8714,137 @@ fi CFLAGS="$LZ4_CFLAGS $CFLAGS" fi +# +# ZSTD +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with zstd support" >&5 +$as_echo_n "checking whether to build with zstd support... " >&6; } + + + +# Check whether --with-zstd was given. +if test "${with_zstd+set}" = set; then : + withval=$with_zstd; + case $withval in + yes) + +$as_echo "#define USE_ZSTD 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-zstd option" "$LINENO" 5 + ;; + esac + +else + with_zstd=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_zstd" >&5 +$as_echo "$with_zstd" >&6; } + + +if test "$with_zstd" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for libzstd" >&5 +$as_echo_n "checking for libzstd... " >&6; } + +if test -n "$ZSTD_CFLAGS"; then + pkg_cv_ZSTD_CFLAGS="$ZSTD_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libzstd\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libzstd") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_ZSTD_CFLAGS=`$PKG_CONFIG --cflags "libzstd" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$ZSTD_LIBS"; then + pkg_cv_ZSTD_LIBS="$ZSTD_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libzstd\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libzstd") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_ZSTD_LIBS=`$PKG_CONFIG --libs "libzstd" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + ZSTD_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libzstd" 2>&1` + else + ZSTD_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libzstd" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$ZSTD_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (libzstd) were not met: + +$ZSTD_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables ZSTD_CFLAGS +and ZSTD_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables ZSTD_CFLAGS +and ZSTD_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + ZSTD_CFLAGS=$pkg_cv_ZSTD_CFLAGS + ZSTD_LIBS=$pkg_cv_ZSTD_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$ZSTD_LIBS $LIBS" + CFLAGS="$ZSTD_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13556,6 +13696,29 @@ done CPPFLAGS=$ac_save_CPPFLAGS fi +if test "$with_zstd" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$ZSTD_CFLAGS $CPPFLAGS" + + # Verify we have zstd's header files + for ac_header in zstd.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "zstd.h" "ac_cv_header_zstd_h" "$ac_includes_default" +if test "x$ac_cv_header_zstd_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_ZSTD_H 1 +_ACEOF + +else + as_fn_error $? "zstd.h header file is required for zstd" "$LINENO" 5 +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index d62ea5f742..abac4b28f6 100644 --- a/configure.ac +++ b/configure.ac @@ -1001,6 +1001,21 @@ if test "$with_lz4" = yes; then CFLAGS="$LZ4_CFLAGS $CFLAGS" fi +# +# ZSTD +# +AC_MSG_CHECKING([whether to build with zstd support]) +PGAC_ARG_BOOL(with, zstd, no, [build with zstd support], + [AC_DEFINE([USE_ZSTD], 1, [Define to 1 to build with zstd support. (--with-zstd)])]) +AC_MSG_RESULT([$with_zstd]) +AC_SUBST(with_zstd) + +if test "$with_zstd" = yes; then + PKG_CHECK_MODULES(ZSTD, libzstd) + LIBS="$ZSTD_LIBS $LIBS" + CFLAGS="$ZSTD_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1433,6 +1448,17 @@ if test "$with_lz4" = yes; then CPPFLAGS=$ac_save_CPPFLAGS fi +if test "$with_zstd" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$ZSTD_CFLAGS $CPPFLAGS" + + # Verify we have zstd's header files + AC_CHECK_HEADERS(zstd.h, [], + [AC_MSG_ERROR([zstd.h header file is required for zstd])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f33355e44a..f2749ebf45 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -189,6 +189,9 @@ const struct config_enum_entry wal_compression_options[] = { #endif #ifdef USE_LZ4 {"lz4", WAL_COMPRESSION_LZ4, false}, +#endif +#ifdef USE_ZSTD + {"zstd", WAL_COMPRESSION_ZSTD, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 38ac5f9e94..852e5ef7af 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,6 +41,10 @@ #include "lz4.h" #endif +#ifdef USE_ZSTD +#include "zstd.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -896,6 +900,22 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, break; #endif +#ifdef USE_ZSTD + case WAL_COMPRESSION_ZSTD: + len = ZSTD_compress(dest, PGLZ_MAX_BLCKSZ, + source, orig_len, ZSTD_CLEVEL_DEFAULT); + if (ZSTD_isError(len)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("failed compressing zstd: %s", + ZSTD_getErrorName(len)))); + len = -1; + } + + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 920c47e4d7..6ce2fbfb13 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -41,6 +41,10 @@ #include "lz4.h" #endif +#ifdef USE_ZSTD +#include "zstd.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1564,6 +1568,8 @@ wal_compression_name(WalCompression compression) return "zlib"; case WAL_COMPRESSION_LZ4: return "lz4"; + case WAL_COMPRESSION_ZSTD: + return "zstd"; default: return "???"; } @@ -1619,6 +1625,15 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) break; #endif +#ifdef USE_ZSTD + case WAL_COMPRESSION_ZSTD: + decomp_result = ZSTD_decompress(tmp.data, BLCKSZ, ptr, bkpb->bimg_len); + // XXX: ZSTD_getErrorName + if (ZSTD_isError(decomp_result)) + decomp_result = -1; + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 0993de784e..30f663f448 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -329,6 +329,7 @@ typedef enum WalCompression WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, WAL_COMPRESSION_LZ4 = 2, + WAL_COMPRESSION_ZSTD = 3, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 0a6422da4f..ad26393352 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -902,6 +902,9 @@ /* Define to 1 to build with LZ4 support (--with-lz4) */ #undef USE_LZ4 +/* Define to 1 if you have the `zstd' library (-lzstd). */ +#undef USE_ZSTD + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index 14605371bb..7dfa0f31f6 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -307,6 +307,7 @@ sub GenerateFiles HAVE_LIBXML2 => undef, HAVE_LIBXSLT => undef, HAVE_LIBZ => $self->{options}->{zlib} ? 1 : undef, + HAVE_LIBZSTD => $self->{options}->{zstd} ? 1 : undef, HAVE_LINK => undef, HAVE_LOCALE_T => 1, HAVE_LONG_INT_64 => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu-- ^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: Remove unnecessary static type qualifiers @ 2025-04-08 12:22 Junwang Zhao <[email protected]> 2025-04-08 14:13 ` Re: Remove unnecessary static type qualifiers Tom Lane <[email protected]> 0 siblings, 1 reply; 3+ messages in thread From: Junwang Zhao @ 2025-04-08 12:22 UTC (permalink / raw) To: Japin Li <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> On Tue, Apr 8, 2025 at 4:29 PM Japin Li <[email protected]> wrote: > > > Hi, all > > When I read the libpq source code, I found unnecessary static type qualifiers > in PQsetClientEncoding(). > > diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c > index 0258d9ace3c..300ddfffd55 100644 > --- a/src/interfaces/libpq/fe-connect.c > +++ b/src/interfaces/libpq/fe-connect.c > @@ -7717,7 +7717,7 @@ int > PQsetClientEncoding(PGconn *conn, const char *encoding) > { > char qbuf[128]; > - static const char query[] = "set client_encoding to '%s'"; > + const char query[] = "set client_encoding to '%s'"; > PGresult *res; > int status; > I doubt that, if you remove the *static*, it will allocate more memory on stack and need more instructions to copy the string to that area. You can check the assembly to prove this. Here is what I got on macos M3 chip. with static: 0000000000015710 <_PQsetClientEncoding>: 15710: d10383ff sub sp, sp, #0xe0 15714: a90d7bfd stp x29, x30, [sp, #0xd0] 15718: 910343fd add x29, sp, #0xd0 1571c: f00002a8 adrp x8, 0x6c000 <_write+0x6c000> 15720: f941a908 ldr x8, [x8, #0x350] 15724: f9400108 ldr x8, [x8] 15728: f81f83a8 stur x8, [x29, #-0x8] 1572c: f9001fe0 str x0, [sp, #0x38] 15730: f9001be1 str x1, [sp, #0x30] 15734: f9401fe8 ldr x8, [sp, #0x38] 15738: f1000108 subs x8, x8, #0x0 1573c: 1a9f17e8 cset w8, eq 15740: 37000108 tbnz w8, #0x0, 0x15760 <_PQsetClientEncoding+0x50> 15744: 14000001 b 0x15748 <_PQsetClientEncoding+0x38> 15748: f9401fe8 ldr x8, [sp, #0x38] 1574c: b941f908 ldr w8, [x8, #0x1f8] without: 00000000000156ec <_PQsetClientEncoding>: 156ec: d10443ff sub sp, sp, #0x110 156f0: a90f6ffc stp x28, x27, [sp, #0xf0] 156f4: a9107bfd stp x29, x30, [sp, #0x100] 156f8: 910403fd add x29, sp, #0x100 156fc: f00002a8 adrp x8, 0x6c000 <_write+0x6c000> 15700: f941a908 ldr x8, [x8, #0x350] 15704: f9400108 ldr x8, [x8] 15708: f81e83a8 stur x8, [x29, #-0x18] 1570c: f9001be0 str x0, [sp, #0x30] 15710: f90017e1 str x1, [sp, #0x28] 15714: f00001c9 adrp x9, 0x50000 <_write+0x50000> 15718: 913ded29 add x9, x9, #0xf7b 1571c: 3dc00120 ldr q0, [x9] 15720: 910103e8 add x8, sp, #0x40 15724: 3d8013e0 str q0, [sp, #0x40] 15728: 3cc0c120 ldur q0, [x9, #0xc] 1572c: 3c80c100 stur q0, [x8, #0xc] 15730: f9401be8 ldr x8, [sp, #0x30] 15734: f1000108 subs x8, x8, #0x0 15738: 1a9f17e8 cset w8, eq 1573c: 37000108 tbnz w8, #0x0, 0x1575c <_PQsetClientEncoding+0x70> 15740: 14000001 b 0x15744 <_PQsetClientEncoding+0x58> 15744: f9401be8 ldr x8, [sp, #0x30] 15748: b941f908 ldr w8, [x8, #0x1f8] > > > -- > Regrads, > Japin Li > > -- Regards Junwang Zhao ^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: Remove unnecessary static type qualifiers 2025-04-08 12:22 Re: Remove unnecessary static type qualifiers Junwang Zhao <[email protected]> @ 2025-04-08 14:13 ` Tom Lane <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Tom Lane @ 2025-04-08 14:13 UTC (permalink / raw) To: Junwang Zhao <[email protected]>; +Cc: Japin Li <[email protected]>; PostgreSQL Hackers <[email protected]> Junwang Zhao <[email protected]> writes: > On Tue, Apr 8, 2025 at 4:29 PM Japin Li <[email protected]> wrote: >> - static const char query[] = "set client_encoding to '%s'"; >> + const char query[] = "set client_encoding to '%s'"; > I doubt that, if you remove the *static*, it will allocate more memory > on stack and need more instructions to copy the string to that area. Yeah, it's not an improvement as written. This might be OK: >> + const char *query = "set client_encoding to '%s'"; But I kinda doubt it's worth touching. regards, tom lane ^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2025-04-08 14:13 UTC | newest] Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-12 20:43 [PATCH 7/8] add wal_compression_method=zstd Justin Pryzby <[email protected]> 2025-04-08 12:22 Re: Remove unnecessary static type qualifiers Junwang Zhao <[email protected]> 2025-04-08 14:13 ` Re: Remove unnecessary static type qualifiers Tom Lane <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox