agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH 05/10] (re)add wal_compression_method: LZ4 137+ messages / 4 participants [nested] [flat]
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/12] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- doc/src/sgml/config.sgml | 3 ++- src/backend/access/transam/xlog.c | 3 +++ src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++- src/backend/access/transam/xlogreader.c | 12 ++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index bed89d055d..1f21e96922 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 3ece007922..23f5bdc335 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 530007f55a..2a2bccbe5c 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); -- 2.17.0 --45Z9DzgjV8m4Oswq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-add-wal_compression_method-zstd.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/12] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- doc/src/sgml/config.sgml | 3 ++- src/backend/access/transam/xlog.c | 3 +++ src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++- src/backend/access/transam/xlogreader.c | 12 ++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index bed89d055d..1f21e96922 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 3ece007922..23f5bdc335 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 530007f55a..2a2bccbe5c 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); -- 2.17.0 --45Z9DzgjV8m4Oswq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-add-wal_compression_method-zstd.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH v7 5/9] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- doc/src/sgml/config.sgml | 3 ++- doc/src/sgml/install-windows.sgml | 2 +- doc/src/sgml/installation.sgml | 5 +++-- src/backend/access/transam/xlog.c | 3 +++ src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++- src/backend/access/transam/xlogreader.c | 12 ++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + 8 files changed, 39 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 70effa6345..f2209f5df0 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3147,7 +3147,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index db53ee85a8..a023584722 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m"; <term><productname>LZ4</productname></term> <listitem><para> Required for supporting <productname>LZ4</productname> compression - method for compressing the table data. Binaries and source can be + method for compressing table or WAL data. Binaries and source can be downloaded from <ulink url="https://github.com/lz4/lz4/releases"></ulink;. </para></listitem> diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 3c0aa118c7..7c8cdac25a 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -270,7 +270,8 @@ su - postgres <para> You need <productname>LZ4</productname>, if you want to support compression of data with this method; see - <xref linkend="guc-default-toast-compression"/>. + <xref linkend="guc-default-toast-compression"/> and + <xref linkend="guc-wal-compression-method"/>. </para> </listitem> @@ -980,7 +981,7 @@ build-postgresql: <para> Build with <productname>LZ4</productname> compression support. This allows the use of <productname>LZ4</productname> for - compression of table data. + compression of table and WAL data. </para> </listitem> </varlistentry> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index a43e04fc0b..999896487e 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 5ab07621d6..30804e8cd5 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index baed4d9228..6421ddbde8 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -219,7 +219,7 @@ #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index ceca0f7189..fc57ea728e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0006-add-wal_compression_method-zstd.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --XsQoSWH+UP9D9v3l Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 17 +- src/backend/access/transam/xlogreader.c | 12 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 241 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 3fd4cecbeb..fed440adcf 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13381,6 +13521,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 2f1585adc0..8c454128bb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f5d4450654..984ce39cc7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index a93b33464f..3c15286bd8 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0d8830fc50..97165f1bb1 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len) struct walcompression walmethods[] = { {"pglz", WAL_COMPRESSION_PGLZ}, {"zlib", WAL_COMPRESSION_ZLIB}, + {"lz4", WAL_COMPRESSION_LZ4}, }; /* @@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b000a21557..b908fa48de 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -336,6 +336,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 12 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 8 files changed, 234 insertions(+) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index ee73bc3afd..38ac5f9e94 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 143df55fcb..920c47e4d7 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b80759ed45..0993de784e 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ = 0, WAL_COMPRESSION_ZLIB = 1, + WAL_COMPRESSION_LZ4 = 2, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --f0KYrhQ4vYSV2aJu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++ configure.ac | 26 +++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 3 + src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 10 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..f33355e44a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ef69a94492..76f494cb9b 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -213,7 +213,7 @@ # open_sync #full_page_writes = on # recover from partial page writes #wal_compression = off # enable compression of full-page writes -#wal_compression_method = pglz # pglz, zlib +#wal_compression_method = pglz # pglz, zlib, lz4 #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) #wal_init_zero = on # zero-fill new WAL files diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-re-add-wal_compression_method-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4 @ 2021-03-12 20:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw) --- configure | 176 ++++++++++++++++++++++++ configure.ac | 26 ++++ doc/src/sgml/config.sgml | 3 +- src/backend/access/transam/xlog.c | 5 +- src/backend/access/transam/xloginsert.c | 12 ++ src/backend/access/transam/xlogreader.c | 13 ++ src/include/access/xlog_internal.h | 1 + src/include/pg_config.h.in | 3 + src/tools/msvc/Solution.pm | 1 + 9 files changed, 238 insertions(+), 2 deletions(-) diff --git a/configure b/configure index fad817bb38..9b590eb432 100755 --- a/configure +++ b/configure @@ -699,6 +699,9 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS +with_lz4 with_zlib with_system_tzdata with_libxslt @@ -864,6 +867,7 @@ with_libxml with_libxslt with_system_tzdata with_zlib +with_lz4 with_gnu_ld with_ssl with_openssl @@ -891,6 +895,8 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL @@ -1569,6 +1575,7 @@ Optional Packages: --with-system-tzdata=DIR use system time zone data in DIR --without-zlib do not use Zlib + --with-lz4 build with LZ4 support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -1596,6 +1603,8 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program @@ -8563,6 +8572,137 @@ fi +# +# LZ4 +# +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5 +$as_echo_n "checking whether to build with LZ4 support... " >&6; } + + + +# Check whether --with-lz4 was given. +if test "${with_lz4+set}" = set; then : + withval=$with_lz4; + case $withval in + yes) + +$as_echo "#define USE_LZ4 1" >>confdefs.h + + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5 + ;; + esac + +else + with_lz4=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5 +$as_echo "$with_lz4" >&6; } + + +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -13378,6 +13518,42 @@ fi fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + for ac_header in lz4/lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_LZ4_H 1 +_ACEOF + +else + for ac_header in lz4.h +do : + ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" +if test "x$ac_cv_header_lz4_h" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LZ4_H 1 +_ACEOF + +else + as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 +fi + +done + +fi + +done + + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then for ac_header in gssapi/gssapi.h do : diff --git a/configure.ac b/configure.ac index 0ed53571dd..2b3b5676eb 100644 --- a/configure.ac +++ b/configure.ac @@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes, [do not use Zlib]) AC_SUBST(with_zlib) +# +# LZ4 +# +AC_MSG_CHECKING([whether to build with LZ4 support]) +PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], + [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])]) +AC_MSG_RESULT([$with_lz4]) +AC_SUBST(with_lz4) + +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi +if test "$with_lz4" = yes; then + ac_save_CPPFLAGS=$CPPFLAGS + CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS" + + # Verify we have LZ4's header files + AC_CHECK_HEADERS(lz4/lz4.h, [], + [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) + + CPPFLAGS=$ac_save_CPPFLAGS +fi + if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 7fb2a84626..257775c83b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3082,7 +3082,8 @@ include_dir 'conf.d' <para> This parameter selects the compression method used to compress WAL when <varname>wal_compression</varname> is enabled. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>pglz</literal>. Only superusers can change this setting. </para> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 8bae73b4ec..609d954a3a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZLIB; +int wal_compression_method = WAL_COMPRESSION_LZ4; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = { {"pglz", WAL_COMPRESSION_PGLZ, false}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 34e1227381..9fe5c30236 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) @@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index afca22a26c..fa1e38a810 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -37,6 +37,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression) return "pglz"; case WAL_COMPRESSION_ZLIB: return "zlib"; + case WAL_COMPRESSION_LZ4: + return "lz4"; default: return "???"; } @@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index d653839b97..e70886b81c 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -328,6 +328,7 @@ typedef enum WalCompression { WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 7a7cc21d8d..0a6422da4f 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -899,6 +899,9 @@ /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */ #undef USE_LLVM +/* Define to 1 to build with LZ4 support (--with-lz4) */ +#undef USE_LZ4 + /* Define to select named POSIX semaphores. */ #undef USE_NAMED_POSIX_SEMAPHORES diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index a4f5cc4bdb..14605371bb 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -485,6 +485,7 @@ sub GenerateFiles USE_ICU => $self->{options}->{icu} ? 1 : undef, USE_LIBXML => undef, USE_LIBXSLT => undef, + USE_LZ4 => undef, USE_LDAP => $self->{options}->{ldap} ? 1 : undef, USE_LLVM => undef, USE_NAMED_POSIX_SEMAPHORES => undef, -- 2.17.0 --0qVF/w3MHQqLSynd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH v9 5/9] (re)add wal_compression_method: lz4 @ 2021-05-25 04:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-05-25 04:32 UTC (permalink / raw) --- doc/src/sgml/config.sgml | 3 ++- doc/src/sgml/install-windows.sgml | 2 +- doc/src/sgml/installation.sgml | 5 +++-- src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++- src/backend/access/transam/xlogreader.c | 15 +++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/access/xlogrecord.h | 2 +- 8 files changed, 40 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 1df56d8034..df5ff70d91 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3140,7 +3140,8 @@ include_dir 'conf.d' server compresses full page images written to WAL when <xref linkend="guc-full-page-writes"/> is on or during a base backup. A compressed page image will be decompressed during WAL replay. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>off</literal>. Only superusers can change this setting. </para> diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index 312edc6f7a..ba794b8c93 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m"; <term><productname>LZ4</productname></term> <listitem><para> Required for supporting <productname>LZ4</productname> compression - method for compressing the table data. Binaries and source can be + method for compressing table or WAL data. Binaries and source can be downloaded from <ulink url="https://github.com/lz4/lz4/releases"></ulink;. </para></listitem> diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 073d5089f7..c7673a4dc8 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -270,7 +270,8 @@ su - postgres <para> You need <productname>LZ4</productname>, if you want to support compression of data with this method; see - <xref linkend="guc-default-toast-compression"/>. + <xref linkend="guc-default-toast-compression"/> and + <xref linkend="guc-wal-compression"/>. </para> </listitem> @@ -980,7 +981,7 @@ build-postgresql: <para> Build with <productname>LZ4</productname> compression support. This allows the use of <productname>LZ4</productname> for - compression of table data. + compression of table and WAL data. </para> </listitem> </varlistentry> diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 4f81f19c49..a8794a941a 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -889,6 +896,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 3a922caaeb..e44817fece 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -38,6 +38,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -75,6 +79,10 @@ const struct config_enum_entry wal_compression_options[] = { {"zlib", WAL_COMPRESSION_ZLIB, false}, #endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, +#endif + {NULL, 0, false} }; @@ -1624,6 +1632,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 61786c6e07..728acef953 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -218,7 +218,7 @@ #full_page_writes = on # recover from partial page writes #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) -#wal_compression = off # enable compression of full-page writes: off, pglz, zlib +#wal_compression = off # enable compression of full-page writes: off, pglz, zlib, lz4 #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 8b740af66d..0287592cd4 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -330,6 +330,7 @@ typedef enum WalCompression WAL_COMPRESSION_NONE, WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index 2a60c0fb92..abb42b364d 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -147,7 +147,7 @@ typedef struct XLogRecordBlockImageHeader #define BKPIMAGE_APPLY 0x02 /* page image should be restored during * replay */ #define BKPIMAGE_COMPRESS_METHOD1 0x04 /* bits to encode compression method */ -#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib */ +#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib, 3=lz4 */ /* How many bits to shift to extract compression */ #define BKPIMAGE_COMPRESS_OFFSET_BITS 2 -- 2.17.0 --neYutvxvOLaeuPCA Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH v8 5/9] (re)add wal_compression_method: lz4 @ 2021-05-25 04:32 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: Justin Pryzby @ 2021-05-25 04:32 UTC (permalink / raw) --- doc/src/sgml/config.sgml | 3 ++- doc/src/sgml/install-windows.sgml | 2 +- doc/src/sgml/installation.sgml | 5 +++-- src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++- src/backend/access/transam/xlogreader.c | 14 ++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 2 +- src/include/access/xlog_internal.h | 1 + src/include/access/xlogrecord.h | 2 +- 8 files changed, 39 insertions(+), 7 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 218a98cfc6..352b31fa81 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3126,7 +3126,8 @@ include_dir 'conf.d' server compresses full page images written to WAL when <xref linkend="guc-full-page-writes"/> is on or during a base backup. A compressed page image will be decompressed during WAL replay. - The supported methods are pglz and zlib. + The supported methods are pglz, zlib, and (if configured when + <productname>PostgreSQL</productname> was built) lz4. The default value is <literal>off</literal>. Only superusers can change this setting. </para> diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml index db53ee85a8..a023584722 100644 --- a/doc/src/sgml/install-windows.sgml +++ b/doc/src/sgml/install-windows.sgml @@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m"; <term><productname>LZ4</productname></term> <listitem><para> Required for supporting <productname>LZ4</productname> compression - method for compressing the table data. Binaries and source can be + method for compressing table or WAL data. Binaries and source can be downloaded from <ulink url="https://github.com/lz4/lz4/releases"></ulink;. </para></listitem> diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml index 073d5089f7..c7673a4dc8 100644 --- a/doc/src/sgml/installation.sgml +++ b/doc/src/sgml/installation.sgml @@ -270,7 +270,8 @@ su - postgres <para> You need <productname>LZ4</productname>, if you want to support compression of data with this method; see - <xref linkend="guc-default-toast-compression"/>. + <xref linkend="guc-default-toast-compression"/> and + <xref linkend="guc-wal-compression"/>. </para> </listitem> @@ -980,7 +981,7 @@ build-postgresql: <para> Build with <productname>LZ4</productname> compression support. This allows the use of <productname>LZ4</productname> for - compression of table data. + compression of table and WAL data. </para> </listitem> </varlistentry> diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 4f81f19c49..a8794a941a 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -41,10 +41,17 @@ #define ZLIB_MAX_BLCKSZ 0 #endif +#ifdef USE_LZ4 +#include "lz4.h" +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ) +#else +#define LZ4_MAX_BLCKSZ 0 +#endif + /* Buffer size required to store a compressed version of backup block image */ #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ) -#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ) +#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ) /* * For each block reference registered with XLogRegisterBuffer, we fill in @@ -889,6 +896,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1); + if (len == 0) + len = -1; + break; +#endif + default: /* * It should be impossible to get here for unsupported algorithms, diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 822b2612cd..753b38e58a 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -38,6 +38,10 @@ #include <zlib.h> #endif +#ifdef USE_LZ4 +#include "lz4.h" +#endif + static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3); static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength); @@ -72,6 +76,9 @@ const struct config_enum_entry wal_compression_options[] = { {"1", WAL_COMPRESSION_PGLZ, true}, #ifdef HAVE_LIBZ {"zlib", WAL_COMPRESSION_ZLIB, false}, +#endif +#ifdef USE_LZ4 + {"lz4", WAL_COMPRESSION_LZ4, false}, #endif {NULL, 0, false} }; @@ -1622,6 +1629,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) } #endif +#ifdef USE_LZ4 + case WAL_COMPRESSION_LZ4: + decomp_result = LZ4_decompress_safe(ptr, tmp.data, + bkpb->bimg_len, BLCKSZ-bkpb->hole_length); + break; +#endif + default: report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)", (uint32) (record->ReadRecPtr >> 32), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 3991d35afd..dbf3155502 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -218,7 +218,7 @@ #full_page_writes = on # recover from partial page writes #wal_log_hints = off # also do full page writes of non-critical updates # (change requires restart) -#wal_compression = off # enable compression of full-page writes: off, pglz, zlib +#wal_compression = off # enable compression of full-page writes: off, pglz, zlib, lz4 #wal_init_zero = on # zero-fill new WAL files #wal_recycle = on # recycle WAL files #wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 8b740af66d..0287592cd4 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -330,6 +330,7 @@ typedef enum WalCompression WAL_COMPRESSION_NONE, WAL_COMPRESSION_PGLZ, WAL_COMPRESSION_ZLIB, + WAL_COMPRESSION_LZ4, } WalCompression; extern const char *wal_compression_name(WalCompression compression); diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h index 2a60c0fb92..abb42b364d 100644 --- a/src/include/access/xlogrecord.h +++ b/src/include/access/xlogrecord.h @@ -147,7 +147,7 @@ typedef struct XLogRecordBlockImageHeader #define BKPIMAGE_APPLY 0x02 /* page image should be restored during * replay */ #define BKPIMAGE_COMPRESS_METHOD1 0x04 /* bits to encode compression method */ -#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib */ +#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib, 3=lz4 */ /* How many bits to shift to extract compression */ #define BKPIMAGE_COMPRESS_OFFSET_BITS 2 -- 2.17.0 --P+33d92oIH25kiaB Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0006-Default-to-LZ4.patch" ^ permalink raw reply [nested|flat] 137+ messages in thread
* Potential issue in ecpg-informix decimal converting functions @ 2024-02-22 16:54 [email protected] 0 siblings, 1 reply; 137+ messages in thread From: [email protected] @ 2024-02-22 16:54 UTC (permalink / raw) To: PostgreSQL Hackers <[email protected]> Hi, everyone! I found a potential bug in dectoint() and dectolong() functions from informix.c. "Informix Compatibility Mode" doc chapter says that ECPG_INFORMIX_NUM_OVERFLOW is returned if an overflow occurred. But check this line in dectoint() or dectolong() (it is present in both): if (ret == PGTYPES_NUM_OVERFLOW) - condition is always false because PGTYPESnumeric_to_int() and PGTYPESnumeric_to_long() functions return only 0 or -1. So ECPG_INFORMIX_NUM_OVERFLOW can never be returned. I think dectoint(), dectolong() and PGTYPESnumeric_to_int() functions should be a little bit different like in proposing patch. What do you think? The flaw was catched with the help of Svace static analyzer. https://svace.pages.ispras.ru/svace-website/en/ Thank you! Attachments: [text/x-diff] informix_convert_from_decimal.patch (5.7K, ../../[email protected]/2-informix_convert_from_decimal.patch) download | inline diff: diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 73351a9136..8390a61cf4 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -8882,7 +8882,7 @@ int dectodbl(decimal *np, double *dblp); <term><function>dectoint</function></term> <listitem> <para> - Convert a variable to type decimal to an integer. + Convert a variable of type decimal to an integer. <synopsis> int dectoint(decimal *np, int *ip); </synopsis> @@ -8892,8 +8892,9 @@ int dectoint(decimal *np, int *ip); </para> <para> On success, 0 is returned and a negative value if the conversion - failed. If an overflow occurred, <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> - is returned. + failed. If overflow or underflow occurred, the function returns + <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> or + <literal>ECPG_INFORMIX_NUM_UNDERFLOW</literal> respectively. </para> <para> Note that the ECPG implementation differs from the <productname>Informix</productname> @@ -8908,7 +8909,7 @@ int dectoint(decimal *np, int *ip); <term><function>dectolong</function></term> <listitem> <para> - Convert a variable to type decimal to a long integer. + Convert a variable of type decimal to a long integer. <synopsis> int dectolong(decimal *np, long *lngp); </synopsis> @@ -8918,8 +8919,9 @@ int dectolong(decimal *np, long *lngp); </para> <para> On success, 0 is returned and a negative value if the conversion - failed. If an overflow occurred, <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> - is returned. + failed. If overflow or underflow occurred, the function returns + <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> or + <literal>ECPG_INFORMIX_NUM_UNDERFLOW</literal> respectively. </para> <para> Note that the ECPG implementation differs from the <productname>Informix</productname> diff --git a/src/interfaces/ecpg/compatlib/informix.c b/src/interfaces/ecpg/compatlib/informix.c index 80d40aa3e0..73e84bc6e3 100644 --- a/src/interfaces/ecpg/compatlib/informix.c +++ b/src/interfaces/ecpg/compatlib/informix.c @@ -435,6 +435,7 @@ dectoint(decimal *np, int *ip) { int ret; numeric *nres = PGTYPESnumeric_new(); + int errnum = 0; if (nres == NULL) return ECPG_INFORMIX_OUT_OF_MEMORY; @@ -445,11 +446,18 @@ dectoint(decimal *np, int *ip) return ECPG_INFORMIX_OUT_OF_MEMORY; } + errno = 0; ret = PGTYPESnumeric_to_int(nres, ip); + errnum = errno; PGTYPESnumeric_free(nres); - if (ret == PGTYPES_NUM_OVERFLOW) - ret = ECPG_INFORMIX_NUM_OVERFLOW; + if (ret) + { + if (errnum == PGTYPES_NUM_UNDERFLOW) + ret = ECPG_INFORMIX_NUM_UNDERFLOW; + else if (errnum == PGTYPES_NUM_OVERFLOW) + ret = ECPG_INFORMIX_NUM_OVERFLOW; + } return ret; } @@ -459,6 +467,7 @@ dectolong(decimal *np, long *lngp) { int ret; numeric *nres = PGTYPESnumeric_new(); + int errnum = 0; if (nres == NULL) return ECPG_INFORMIX_OUT_OF_MEMORY; @@ -469,11 +478,18 @@ dectolong(decimal *np, long *lngp) return ECPG_INFORMIX_OUT_OF_MEMORY; } + errno = 0; ret = PGTYPESnumeric_to_long(nres, lngp); + errnum = errno; PGTYPESnumeric_free(nres); - if (ret == PGTYPES_NUM_OVERFLOW) - ret = ECPG_INFORMIX_NUM_OVERFLOW; + if (ret) + { + if (errnum == PGTYPES_NUM_UNDERFLOW) + ret = ECPG_INFORMIX_NUM_UNDERFLOW; + else if (errnum == PGTYPES_NUM_OVERFLOW) + ret = ECPG_INFORMIX_NUM_OVERFLOW; + } return ret; } diff --git a/src/interfaces/ecpg/pgtypeslib/numeric.c b/src/interfaces/ecpg/pgtypeslib/numeric.c index 35e7b92da4..52b49e1ce9 100644 --- a/src/interfaces/ecpg/pgtypeslib/numeric.c +++ b/src/interfaces/ecpg/pgtypeslib/numeric.c @@ -1502,7 +1502,12 @@ PGTYPESnumeric_to_int(numeric *nv, int *ip) /* silence compilers that might complain about useless tests */ #if SIZEOF_LONG > SIZEOF_INT - if (l < INT_MIN || l > INT_MAX) + if (l < INT_MIN) + { + errno = PGTYPES_NUM_UNDERFLOW; + return -1; + } + else if(l > INT_MAX) { errno = PGTYPES_NUM_OVERFLOW; return -1; diff --git a/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout b/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout index 1f8675b3f3..71a5afa4a7 100644 --- a/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout +++ b/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout @@ -3,8 +3,8 @@ (no errno set) - dec[0,3]: r: -1, * (no errno set) - dec[0,4]: r: -1, * dec[0,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[0,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[0,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[0,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[0,8]: 0 (r: -1200) (errno == PGTYPES_NUM_OVERFLOW) - dec[0,10]: 0 (r: -1) dec[1,1]: r: 0, -2 @@ -45,8 +45,8 @@ dec[4,2]: r: 0, 592490000000000000000000 dec[4,3]: r: 0, 592490000000000000000000.0 dec[4,4]: r: 0, 592490000000000000000000.00 dec[4,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[4,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[4,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[4,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[4,8]: 0 (r: -1200) dec[4,10]: 5.9249e+23 (r: 0) dec[5,1]: r: 0, -328400 @@ -141,8 +141,8 @@ dec[13,2]: r: 0, 1234567890123456789012345679 dec[13,3]: r: 0, 1234567890123456789012345678.9 dec[13,4]: r: 0, 1234567890123456789012345678.91 dec[13,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[13,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[13,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[13,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[13,8]: 0 (r: -1200) dec[13,10]: 1.23457e+27 (r: 0) (errno == PGTYPES_NUM_OVERFLOW) - dec[14,0]: r: -1200 ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-23 10:44 Daniel Gustafsson <[email protected]> parent: [email protected] 0 siblings, 1 reply; 137+ messages in thread From: Daniel Gustafsson @ 2024-02-23 10:44 UTC (permalink / raw) To: [email protected]; +Cc: PostgreSQL Hackers <[email protected]> > On 22 Feb 2024, at 17:54, [email protected] wrote: > PGTYPESnumeric_to_int() and PGTYPESnumeric_to_long() > functions return only 0 or -1. So ECPG_INFORMIX_NUM_OVERFLOW can never > be returned. Indeed, this looks like an oversight. > I think dectoint(), dectolong() and PGTYPESnumeric_to_int() functions > should be a little bit different like in proposing patch. > What do you think? - Convert a variable to type decimal to an integer. + Convert a variable of type decimal to an integer. While related, this should be committed and backpatched regardless. + int errnum = 0; Stylistic nit, we typically don't initialize a variable which cannot be accessed before being set. Overall the patch looks sane, please register it for the next commitfest to make it's not missed. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-23 15:03 [email protected] parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 137+ messages in thread From: [email protected] @ 2024-02-23 15:03 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> Daniel Gustafsson писал(а) 2024-02-23 13:44: >> On 22 Feb 2024, at 17:54, [email protected] wrote: > >> PGTYPESnumeric_to_int() and PGTYPESnumeric_to_long() >> functions return only 0 or -1. So ECPG_INFORMIX_NUM_OVERFLOW can never >> be returned. > > Indeed, this looks like an oversight. > >> I think dectoint(), dectolong() and PGTYPESnumeric_to_int() functions >> should be a little bit different like in proposing patch. >> What do you think? > > - Convert a variable to type decimal to an integer. > + Convert a variable of type decimal to an integer. > While related, this should be committed and backpatched regardless. > > + int errnum = 0; > Stylistic nit, we typically don't initialize a variable which cannot be > accessed before being set. > > Overall the patch looks sane, please register it for the next > commitfest to > make it's not missed. > > -- > Daniel Gustafsson Thank you for feedback, - Convert a variable to type decimal to an integer. + Convert a variable of type decimal to an integer. I removed this from the patch and proposed to [email protected] + int errnum = 0; fixed Thank's for advice, the patch will be registered for the next commitfest. -- Aidar Imamov Attachments: [text/x-diff] informix_convert_from_decimal.patch (5.0K, ../../[email protected]/2-informix_convert_from_decimal.patch) download | inline diff: diff --git a/doc/src/sgml/ecpg.sgml b/doc/src/sgml/ecpg.sgml index 73351a9136..aa86afa1d9 100644 --- a/doc/src/sgml/ecpg.sgml +++ b/doc/src/sgml/ecpg.sgml @@ -8892,8 +8892,9 @@ int dectoint(decimal *np, int *ip); </para> <para> On success, 0 is returned and a negative value if the conversion - failed. If an overflow occurred, <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> - is returned. + failed. If overflow or underflow occurred, the function returns + <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> or + <literal>ECPG_INFORMIX_NUM_UNDERFLOW</literal> respectively. </para> <para> Note that the ECPG implementation differs from the <productname>Informix</productname> @@ -8918,8 +8919,9 @@ int dectolong(decimal *np, long *lngp); </para> <para> On success, 0 is returned and a negative value if the conversion - failed. If an overflow occurred, <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> - is returned. + failed. If overflow or underflow occurred, the function returns + <literal>ECPG_INFORMIX_NUM_OVERFLOW</literal> or + <literal>ECPG_INFORMIX_NUM_UNDERFLOW</literal> respectively. </para> <para> Note that the ECPG implementation differs from the <productname>Informix</productname> diff --git a/src/interfaces/ecpg/compatlib/informix.c b/src/interfaces/ecpg/compatlib/informix.c index 80d40aa3e0..5f9d71d0f3 100644 --- a/src/interfaces/ecpg/compatlib/informix.c +++ b/src/interfaces/ecpg/compatlib/informix.c @@ -434,6 +434,7 @@ int dectoint(decimal *np, int *ip) { int ret; + int errnum; numeric *nres = PGTYPESnumeric_new(); if (nres == NULL) @@ -445,11 +446,18 @@ dectoint(decimal *np, int *ip) return ECPG_INFORMIX_OUT_OF_MEMORY; } + errno = 0; ret = PGTYPESnumeric_to_int(nres, ip); + errnum = errno; PGTYPESnumeric_free(nres); - if (ret == PGTYPES_NUM_OVERFLOW) - ret = ECPG_INFORMIX_NUM_OVERFLOW; + if (ret) + { + if (errnum == PGTYPES_NUM_UNDERFLOW) + ret = ECPG_INFORMIX_NUM_UNDERFLOW; + else if (errnum == PGTYPES_NUM_OVERFLOW) + ret = ECPG_INFORMIX_NUM_OVERFLOW; + } return ret; } @@ -458,6 +466,7 @@ int dectolong(decimal *np, long *lngp) { int ret; + int errnum; numeric *nres = PGTYPESnumeric_new(); if (nres == NULL) @@ -469,11 +478,18 @@ dectolong(decimal *np, long *lngp) return ECPG_INFORMIX_OUT_OF_MEMORY; } + errno = 0; ret = PGTYPESnumeric_to_long(nres, lngp); + errnum = errno; PGTYPESnumeric_free(nres); - if (ret == PGTYPES_NUM_OVERFLOW) - ret = ECPG_INFORMIX_NUM_OVERFLOW; + if (ret) + { + if (errnum == PGTYPES_NUM_UNDERFLOW) + ret = ECPG_INFORMIX_NUM_UNDERFLOW; + else if (errnum == PGTYPES_NUM_OVERFLOW) + ret = ECPG_INFORMIX_NUM_OVERFLOW; + } return ret; } diff --git a/src/interfaces/ecpg/pgtypeslib/numeric.c b/src/interfaces/ecpg/pgtypeslib/numeric.c index 35e7b92da4..52b49e1ce9 100644 --- a/src/interfaces/ecpg/pgtypeslib/numeric.c +++ b/src/interfaces/ecpg/pgtypeslib/numeric.c @@ -1502,7 +1502,12 @@ PGTYPESnumeric_to_int(numeric *nv, int *ip) /* silence compilers that might complain about useless tests */ #if SIZEOF_LONG > SIZEOF_INT - if (l < INT_MIN || l > INT_MAX) + if (l < INT_MIN) + { + errno = PGTYPES_NUM_UNDERFLOW; + return -1; + } + else if(l > INT_MAX) { errno = PGTYPES_NUM_OVERFLOW; return -1; diff --git a/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout b/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout index 1f8675b3f3..71a5afa4a7 100644 --- a/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout +++ b/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout @@ -3,8 +3,8 @@ (no errno set) - dec[0,3]: r: -1, * (no errno set) - dec[0,4]: r: -1, * dec[0,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[0,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[0,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[0,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[0,8]: 0 (r: -1200) (errno == PGTYPES_NUM_OVERFLOW) - dec[0,10]: 0 (r: -1) dec[1,1]: r: 0, -2 @@ -45,8 +45,8 @@ dec[4,2]: r: 0, 592490000000000000000000 dec[4,3]: r: 0, 592490000000000000000000.0 dec[4,4]: r: 0, 592490000000000000000000.00 dec[4,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[4,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[4,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[4,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[4,8]: 0 (r: -1200) dec[4,10]: 5.9249e+23 (r: 0) dec[5,1]: r: 0, -328400 @@ -141,8 +141,8 @@ dec[13,2]: r: 0, 1234567890123456789012345679 dec[13,3]: r: 0, 1234567890123456789012345678.9 dec[13,4]: r: 0, 1234567890123456789012345678.91 dec[13,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[13,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[13,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[13,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[13,8]: 0 (r: -1200) dec[13,10]: 1.23457e+27 (r: 0) (errno == PGTYPES_NUM_OVERFLOW) - dec[14,0]: r: -1200 ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-24 01:15 Michael Paquier <[email protected]> parent: [email protected] 0 siblings, 1 reply; 137+ messages in thread From: Michael Paquier @ 2024-02-24 01:15 UTC (permalink / raw) To: [email protected]; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Hackers <[email protected]> On Fri, Feb 23, 2024 at 06:03:41PM +0300, [email protected] wrote: > Thank's for advice, the patch will be registered for the next commitfest. The risk looks really minimal to me, but playing with error codes while the logic of the function is unchanged does not strike me as something to backpatch as it could slightly break applications. On HEAD, I'm OK with that. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-25 23:28 Daniel Gustafsson <[email protected]> parent: Michael Paquier <[email protected]> 0 siblings, 1 reply; 137+ messages in thread From: Daniel Gustafsson @ 2024-02-25 23:28 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: [email protected]; PostgreSQL Hackers <[email protected]> > On 24 Feb 2024, at 02:15, Michael Paquier <[email protected]> wrote: > > On Fri, Feb 23, 2024 at 06:03:41PM +0300, [email protected] wrote: >> Thank's for advice, the patch will be registered for the next commitfest. > > The risk looks really minimal to me, but playing with error codes > while the logic of the function is unchanged does not strike me as > something to backpatch as it could slightly break applications. On > HEAD, I'm OK with that. Yeah, I think this is for HEAD only, especially given the lack of complaints against backbranches. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-27 05:08 Michael Paquier <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 2 replies; 137+ messages in thread From: Michael Paquier @ 2024-02-27 05:08 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: [email protected]; PostgreSQL Hackers <[email protected]> On Mon, Feb 26, 2024 at 12:28:51AM +0100, Daniel Gustafsson wrote: > Yeah, I think this is for HEAD only, especially given the lack of complaints > against backbranches. Daniel, are you planning to look at that? I haven't done any detailed lookup, but would be happy to do so it that helps. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-27 08:24 Daniel Gustafsson <[email protected]> parent: Michael Paquier <[email protected]> 1 sibling, 1 reply; 137+ messages in thread From: Daniel Gustafsson @ 2024-02-27 08:24 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: [email protected]; PostgreSQL Hackers <[email protected]> > On 27 Feb 2024, at 06:08, Michael Paquier <[email protected]> wrote: > > On Mon, Feb 26, 2024 at 12:28:51AM +0100, Daniel Gustafsson wrote: >> Yeah, I think this is for HEAD only, especially given the lack of complaints >> against backbranches. > > Daniel, are you planning to look at that? I haven't done any detailed > lookup, but would be happy to do so it that helps. I have it on my TODO for the upcoming CF. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-27 23:14 Michael Paquier <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 137+ messages in thread From: Michael Paquier @ 2024-02-27 23:14 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: [email protected]; PostgreSQL Hackers <[email protected]> On Tue, Feb 27, 2024 at 09:24:25AM +0100, Daniel Gustafsson wrote: > I have it on my TODO for the upcoming CF. Okay, thanks. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-02-29 09:43 [email protected] parent: Michael Paquier <[email protected]> 0 siblings, 0 replies; 137+ messages in thread From: [email protected] @ 2024-02-29 09:43 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Hackers <[email protected]> Michael Paquier писал(а) 2024-02-28 02:14: > On Tue, Feb 27, 2024 at 09:24:25AM +0100, Daniel Gustafsson wrote: >> I have it on my TODO for the upcoming CF. > > Okay, thanks. > -- > Michael Greetings! Sorry, I had been waiting for a few days for my cool-off period to end. The patch now is registered to CF in the 'Refactoring' topic. -- Aidar ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-03-06 15:03 Daniel Gustafsson <[email protected]> parent: Michael Paquier <[email protected]> 1 sibling, 1 reply; 137+ messages in thread From: Daniel Gustafsson @ 2024-03-06 15:03 UTC (permalink / raw) To: Michael Paquier <[email protected]>; +Cc: [email protected]; PostgreSQL Hackers <[email protected]> > On 27 Feb 2024, at 06:08, Michael Paquier <[email protected]> wrote: > > On Mon, Feb 26, 2024 at 12:28:51AM +0100, Daniel Gustafsson wrote: >> Yeah, I think this is for HEAD only, especially given the lack of complaints >> against backbranches. > > Daniel, are you planning to look at that? I haven't done any detailed > lookup, but would be happy to do so it that helps. I had a look at this today and opted for trimming back the patch a bit. Reading the informix docs the functions we are mimicking for compatibility here does not have an underflow returnvalue, so adding one doesn't seem right (or helpful). The attached fixes the return of overflow and leaves it at that, which makes it possible to backpatch since it's fixing the code to match the documented behavior. -- Daniel Gustafsson Attachments: [application/octet-stream] v3-0001-ecpg-Fix-return-code-for-overflow-in-numeric-conv.patch (3.7K, ../../[email protected]/2-v3-0001-ecpg-Fix-return-code-for-overflow-in-numeric-conv.patch) download | inline diff: From 4accb63b937357c06271e7b137dc9a6fe3baff01 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Wed, 6 Mar 2024 15:42:26 +0100 Subject: [PATCH v3] ecpg: Fix return code for overflow in numeric conversion The decimal conversion functions dectoint and dectolong are documented to return ECPG_INFORMIX_NUM_OVERFLOW in case of overflows, but always returned -1 on all errors due to incorrectly checking the returnvalue from the PGTYPES* functions. Author: Aidar Imamov <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/interfaces/ecpg/compatlib/informix.c | 10 ++++++++-- .../test/expected/compat_informix-dec_test.stdout | 12 ++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/interfaces/ecpg/compatlib/informix.c b/src/interfaces/ecpg/compatlib/informix.c index 80d40aa3e0..8ea89e640a 100644 --- a/src/interfaces/ecpg/compatlib/informix.c +++ b/src/interfaces/ecpg/compatlib/informix.c @@ -435,6 +435,7 @@ dectoint(decimal *np, int *ip) { int ret; numeric *nres = PGTYPESnumeric_new(); + int errnum; if (nres == NULL) return ECPG_INFORMIX_OUT_OF_MEMORY; @@ -445,10 +446,12 @@ dectoint(decimal *np, int *ip) return ECPG_INFORMIX_OUT_OF_MEMORY; } + errno = 0; ret = PGTYPESnumeric_to_int(nres, ip); + errnum = errno; PGTYPESnumeric_free(nres); - if (ret == PGTYPES_NUM_OVERFLOW) + if (ret == -1 && errnum == PGTYPES_NUM_OVERFLOW) ret = ECPG_INFORMIX_NUM_OVERFLOW; return ret; @@ -459,6 +462,7 @@ dectolong(decimal *np, long *lngp) { int ret; numeric *nres = PGTYPESnumeric_new(); + int errnum; if (nres == NULL) return ECPG_INFORMIX_OUT_OF_MEMORY; @@ -469,10 +473,12 @@ dectolong(decimal *np, long *lngp) return ECPG_INFORMIX_OUT_OF_MEMORY; } + errno = 0; ret = PGTYPESnumeric_to_long(nres, lngp); + errnum = errno; PGTYPESnumeric_free(nres); - if (ret == PGTYPES_NUM_OVERFLOW) + if (ret == -1 && errnum == PGTYPES_NUM_OVERFLOW) ret = ECPG_INFORMIX_NUM_OVERFLOW; return ret; diff --git a/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout b/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout index 1f8675b3f3..71a5afa4a7 100644 --- a/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout +++ b/src/interfaces/ecpg/test/expected/compat_informix-dec_test.stdout @@ -3,8 +3,8 @@ (no errno set) - dec[0,3]: r: -1, * (no errno set) - dec[0,4]: r: -1, * dec[0,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[0,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[0,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[0,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[0,8]: 0 (r: -1200) (errno == PGTYPES_NUM_OVERFLOW) - dec[0,10]: 0 (r: -1) dec[1,1]: r: 0, -2 @@ -45,8 +45,8 @@ dec[4,2]: r: 0, 592490000000000000000000 dec[4,3]: r: 0, 592490000000000000000000.0 dec[4,4]: r: 0, 592490000000000000000000.00 dec[4,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[4,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[4,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[4,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[4,8]: 0 (r: -1200) dec[4,10]: 5.9249e+23 (r: 0) dec[5,1]: r: 0, -328400 @@ -141,8 +141,8 @@ dec[13,2]: r: 0, 1234567890123456789012345679 dec[13,3]: r: 0, 1234567890123456789012345678.9 dec[13,4]: r: 0, 1234567890123456789012345678.91 dec[13,5]: r: 0, 0.00 -(errno == PGTYPES_NUM_OVERFLOW) - dec[13,6]: 0 (r: -1) -(errno == PGTYPES_NUM_OVERFLOW) - dec[13,8]: 0 (r: -1) +(errno == PGTYPES_NUM_OVERFLOW) - dec[13,6]: 0 (r: -1200) +(errno == PGTYPES_NUM_OVERFLOW) - dec[13,8]: 0 (r: -1200) dec[13,10]: 1.23457e+27 (r: 0) (errno == PGTYPES_NUM_OVERFLOW) - dec[14,0]: r: -1200 -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-03-06 19:12 [email protected] parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 137+ messages in thread From: [email protected] @ 2024-03-06 19:12 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: Michael Paquier <[email protected]>; PostgreSQL Hackers <[email protected]> Daniel Gustafsson писал(а) 2024-03-06 18:03: >> On 27 Feb 2024, at 06:08, Michael Paquier <[email protected]> wrote: >> >> On Mon, Feb 26, 2024 at 12:28:51AM +0100, Daniel Gustafsson wrote: >>> Yeah, I think this is for HEAD only, especially given the lack of >>> complaints >>> against backbranches. >> >> Daniel, are you planning to look at that? I haven't done any detailed >> lookup, but would be happy to do so it that helps. > > I had a look at this today and opted for trimming back the patch a bit. > Reading the informix docs the functions we are mimicking for > compatibility here > does not have an underflow returnvalue, so adding one doesn't seem > right (or > helpful). The attached fixes the return of overflow and leaves it at > that, > which makes it possible to backpatch since it's fixing the code to > match the > documented behavior. > > -- > Daniel Gustafsson I agree with the proposed changes in favor of backward compatibility. Also, is it a big deal that the PGTYPESnumeric_to_long() function doesn't exactly match the documentation, compared to PGTYPESnumeric_to_int()? It handles underflow case separately and sets errno to PGTYPES_NUM_UNDERFLOW additionally. ^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Potential issue in ecpg-informix decimal converting functions @ 2024-03-25 13:43 Daniel Gustafsson <[email protected]> parent: [email protected] 0 siblings, 0 replies; 137+ messages in thread From: Daniel Gustafsson @ 2024-03-25 13:43 UTC (permalink / raw) To: [email protected]; +Cc: Michael Paquier <[email protected]>; PostgreSQL Hackers <[email protected]> > On 6 Mar 2024, at 20:12, [email protected] wrote: > I agree with the proposed changes in favor of backward compatibility. I went ahead to pushed this after another look. I'm a bit hesitant to backpatch this since there are no reports against it, and I don't have good sense for how ECPG code is tested and maintained across minor version upgrades. If we want to I will of course do so, so please chime in in case there are different and more informed opinions. > Also, is it a big deal that the PGTYPESnumeric_to_long() function doesn't > exactly match the documentation, compared to PGTYPESnumeric_to_int()? It > handles underflow case separately and sets errno to PGTYPES_NUM_UNDERFLOW > additionally. Fixed as well. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 137+ messages in thread
end of thread, other threads:[~2024-03-25 13:43 UTC | newest] Thread overview: 137+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH v7 5/9] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/12] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/12] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-05-25 04:32 [PATCH v8 5/9] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2021-05-25 04:32 [PATCH v9 5/9] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]> 2024-02-22 16:54 Potential issue in ecpg-informix decimal converting functions [email protected] 2024-02-23 10:44 ` Re: Potential issue in ecpg-informix decimal converting functions Daniel Gustafsson <[email protected]> 2024-02-23 15:03 ` Re: Potential issue in ecpg-informix decimal converting functions [email protected] 2024-02-24 01:15 ` Re: Potential issue in ecpg-informix decimal converting functions Michael Paquier <[email protected]> 2024-02-25 23:28 ` Re: Potential issue in ecpg-informix decimal converting functions Daniel Gustafsson <[email protected]> 2024-02-27 05:08 ` Re: Potential issue in ecpg-informix decimal converting functions Michael Paquier <[email protected]> 2024-02-27 08:24 ` Re: Potential issue in ecpg-informix decimal converting functions Daniel Gustafsson <[email protected]> 2024-02-27 23:14 ` Re: Potential issue in ecpg-informix decimal converting functions Michael Paquier <[email protected]> 2024-02-29 09:43 ` Re: Potential issue in ecpg-informix decimal converting functions [email protected] 2024-03-06 15:03 ` Re: Potential issue in ecpg-informix decimal converting functions Daniel Gustafsson <[email protected]> 2024-03-06 19:12 ` Re: Potential issue in ecpg-informix decimal converting functions [email protected] 2024-03-25 13:43 ` Re: Potential issue in ecpg-informix decimal converting functions Daniel Gustafsson <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox