($INBOX_DIR/description missing)
help / color / mirror / Atom feed[PATCH 05/12] (re)add wal_compression_method: lz4
137+ messages / 4 participants
[nested] [flat]
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/12] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
doc/src/sgml/config.sgml | 3 ++-
src/backend/access/transam/xlog.c | 3 +++
src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++-
src/backend/access/transam/xlogreader.c | 12 ++++++++++++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
6 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bed89d055d..1f21e96922 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3ece007922..23f5bdc335 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 530007f55a..2a2bccbe5c 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
--
2.17.0
--45Z9DzgjV8m4Oswq
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0006-add-wal_compression_method-zstd.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH v7 5/9] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
doc/src/sgml/config.sgml | 3 ++-
doc/src/sgml/install-windows.sgml | 2 +-
doc/src/sgml/installation.sgml | 5 +++--
src/backend/access/transam/xlog.c | 3 +++
src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++-
src/backend/access/transam/xlogreader.c | 12 ++++++++++++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
8 files changed, 39 insertions(+), 6 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 70effa6345..f2209f5df0 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3147,7 +3147,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml
index db53ee85a8..a023584722 100644
--- a/doc/src/sgml/install-windows.sgml
+++ b/doc/src/sgml/install-windows.sgml
@@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m";
<term><productname>LZ4</productname></term>
<listitem><para>
Required for supporting <productname>LZ4</productname> compression
- method for compressing the table data. Binaries and source can be
+ method for compressing table or WAL data. Binaries and source can be
downloaded from
<ulink url="https://github.com/lz4/lz4/releases"></ulink;.
</para></listitem>
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 3c0aa118c7..7c8cdac25a 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -270,7 +270,8 @@ su - postgres
<para>
You need <productname>LZ4</productname>, if you want to support
compression of data with this method; see
- <xref linkend="guc-default-toast-compression"/>.
+ <xref linkend="guc-default-toast-compression"/> and
+ <xref linkend="guc-wal-compression-method"/>.
</para>
</listitem>
@@ -980,7 +981,7 @@ build-postgresql:
<para>
Build with <productname>LZ4</productname> compression support.
This allows the use of <productname>LZ4</productname> for
- compression of table data.
+ compression of table and WAL data.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a43e04fc0b..999896487e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 5ab07621d6..30804e8cd5 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index baed4d9228..6421ddbde8 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -219,7 +219,7 @@
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_init_zero = on # zero-fill new WAL files
#wal_recycle = on # recycle WAL files
#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index ceca0f7189..fc57ea728e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
--
2.17.0
--JYK4vJDZwFMowpUq
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v7-0006-add-wal_compression_method-zstd.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/12] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
doc/src/sgml/config.sgml | 3 ++-
src/backend/access/transam/xlog.c | 3 +++
src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++-
src/backend/access/transam/xlogreader.c | 12 ++++++++++++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
6 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bed89d055d..1f21e96922 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3ece007922..23f5bdc335 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 530007f55a..2a2bccbe5c 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
--
2.17.0
--45Z9DzgjV8m4Oswq
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0006-add-wal_compression_method-zstd.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 5 +-
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
9 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool EnableHotStandby = false;
bool fullPageWrites = true;
bool wal_log_hints = false;
bool wal_compression = false;
-int wal_compression_method = WAL_COMPRESSION_ZLIB;
+int wal_compression_method = WAL_COMPRESSION_LZ4;
char *wal_consistency_checking_string = NULL;
bool *wal_consistency_checking = NULL;
bool wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 17 +-
src/backend/access/transam/xlogreader.c | 12 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 241 insertions(+), 3 deletions(-)
diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13381,6 +13521,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1410,6 +1425,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
struct walcompression walmethods[] = {
{"pglz", WAL_COMPRESSION_PGLZ},
{"zlib", WAL_COMPRESSION_ZLIB},
+ {"lz4", WAL_COMPRESSION_LZ4},
};
/*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++++++++
configure.ac | 26 ++++
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 12 ++
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
8 files changed, 234 insertions(+)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ = 0,
WAL_COMPRESSION_ZLIB = 1,
+ WAL_COMPRESSION_LZ4 = 2,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)
---
configure | 176 ++++++++++++++++++
configure.ac | 26 +++
doc/src/sgml/config.sgml | 3 +-
src/backend/access/transam/xlog.c | 3 +
src/backend/access/transam/xloginsert.c | 12 ++
src/backend/access/transam/xlogreader.c | 13 ++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/pg_config.h.in | 3 +
src/tools/msvc/Solution.pm | 1 +
10 files changed, 238 insertions(+), 2 deletions(-)
diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8563,6 +8572,137 @@ fi
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -13378,6 +13518,42 @@ fi
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
[do not use Zlib])
AC_SUBST(with_zlib)
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
#
# Assignments
#
@@ -1407,6 +1422,17 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ ac_save_CPPFLAGS=$CPPFLAGS
+ CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+ # Verify we have LZ4's header files
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+ CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
<para>
This parameter selects the compression method used to compress WAL when
<varname>wal_compression</varname> is enabled.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>pglz</literal>.
Only superusers can change this setting.
</para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"pglz", WAL_COMPRESSION_PGLZ, false},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
return "pglz";
case WAL_COMPRESSION_ZLIB:
return "zlib";
+ case WAL_COMPRESSION_LZ4:
+ return "lz4";
default:
return "???";
}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
# open_sync
#full_page_writes = on # recover from partial page writes
#wal_compression = off # enable compression of full-page writes
-#wal_compression_method = pglz # pglz, zlib
+#wal_compression_method = pglz # pglz, zlib, lz4
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
#wal_init_zero = on # zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
{
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
2.17.0
--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0005-re-add-wal_compression_method-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH v9 5/9] (re)add wal_compression_method: lz4
@ 2021-05-25 04:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-05-25 04:32 UTC (permalink / raw)
---
doc/src/sgml/config.sgml | 3 ++-
doc/src/sgml/install-windows.sgml | 2 +-
doc/src/sgml/installation.sgml | 5 +++--
src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++-
src/backend/access/transam/xlogreader.c | 15 +++++++++++++++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/access/xlogrecord.h | 2 +-
8 files changed, 40 insertions(+), 7 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 1df56d8034..df5ff70d91 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3140,7 +3140,8 @@ include_dir 'conf.d'
server compresses full page images written to WAL when
<xref linkend="guc-full-page-writes"/> is on or during a base backup.
A compressed page image will be decompressed during WAL replay.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>off</literal>.
Only superusers can change this setting.
</para>
diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml
index 312edc6f7a..ba794b8c93 100644
--- a/doc/src/sgml/install-windows.sgml
+++ b/doc/src/sgml/install-windows.sgml
@@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m";
<term><productname>LZ4</productname></term>
<listitem><para>
Required for supporting <productname>LZ4</productname> compression
- method for compressing the table data. Binaries and source can be
+ method for compressing table or WAL data. Binaries and source can be
downloaded from
<ulink url="https://github.com/lz4/lz4/releases"></ulink;.
</para></listitem>
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 073d5089f7..c7673a4dc8 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -270,7 +270,8 @@ su - postgres
<para>
You need <productname>LZ4</productname>, if you want to support
compression of data with this method; see
- <xref linkend="guc-default-toast-compression"/>.
+ <xref linkend="guc-default-toast-compression"/> and
+ <xref linkend="guc-wal-compression"/>.
</para>
</listitem>
@@ -980,7 +981,7 @@ build-postgresql:
<para>
Build with <productname>LZ4</productname> compression support.
This allows the use of <productname>LZ4</productname> for
- compression of table data.
+ compression of table and WAL data.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 4f81f19c49..a8794a941a 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -889,6 +896,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 3a922caaeb..e44817fece 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -38,6 +38,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -75,6 +79,10 @@ const struct config_enum_entry wal_compression_options[] = {
{"zlib", WAL_COMPRESSION_ZLIB, false},
#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
+#endif
+
{NULL, 0, false}
};
@@ -1624,6 +1632,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 61786c6e07..728acef953 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -218,7 +218,7 @@
#full_page_writes = on # recover from partial page writes
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
-#wal_compression = off # enable compression of full-page writes: off, pglz, zlib
+#wal_compression = off # enable compression of full-page writes: off, pglz, zlib, lz4
#wal_init_zero = on # zero-fill new WAL files
#wal_recycle = on # recycle WAL files
#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 8b740af66d..0287592cd4 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -330,6 +330,7 @@ typedef enum WalCompression
WAL_COMPRESSION_NONE,
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h
index 2a60c0fb92..abb42b364d 100644
--- a/src/include/access/xlogrecord.h
+++ b/src/include/access/xlogrecord.h
@@ -147,7 +147,7 @@ typedef struct XLogRecordBlockImageHeader
#define BKPIMAGE_APPLY 0x02 /* page image should be restored during
* replay */
#define BKPIMAGE_COMPRESS_METHOD1 0x04 /* bits to encode compression method */
-#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib */
+#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib, 3=lz4 */
/* How many bits to shift to extract compression */
#define BKPIMAGE_COMPRESS_OFFSET_BITS 2
--
2.17.0
--neYutvxvOLaeuPCA
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v9-0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* [PATCH v8 5/9] (re)add wal_compression_method: lz4
@ 2021-05-25 04:32 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Justin Pryzby @ 2021-05-25 04:32 UTC (permalink / raw)
---
doc/src/sgml/config.sgml | 3 ++-
doc/src/sgml/install-windows.sgml | 2 +-
doc/src/sgml/installation.sgml | 5 +++--
src/backend/access/transam/xloginsert.c | 17 ++++++++++++++++-
src/backend/access/transam/xlogreader.c | 14 ++++++++++++++
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/include/access/xlog_internal.h | 1 +
src/include/access/xlogrecord.h | 2 +-
8 files changed, 39 insertions(+), 7 deletions(-)
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 218a98cfc6..352b31fa81 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3126,7 +3126,8 @@ include_dir 'conf.d'
server compresses full page images written to WAL when
<xref linkend="guc-full-page-writes"/> is on or during a base backup.
A compressed page image will be decompressed during WAL replay.
- The supported methods are pglz and zlib.
+ The supported methods are pglz, zlib, and (if configured when
+ <productname>PostgreSQL</productname> was built) lz4.
The default value is <literal>off</literal>.
Only superusers can change this setting.
</para>
diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml
index db53ee85a8..a023584722 100644
--- a/doc/src/sgml/install-windows.sgml
+++ b/doc/src/sgml/install-windows.sgml
@@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m";
<term><productname>LZ4</productname></term>
<listitem><para>
Required for supporting <productname>LZ4</productname> compression
- method for compressing the table data. Binaries and source can be
+ method for compressing table or WAL data. Binaries and source can be
downloaded from
<ulink url="https://github.com/lz4/lz4/releases"></ulink;.
</para></listitem>
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 073d5089f7..c7673a4dc8 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -270,7 +270,8 @@ su - postgres
<para>
You need <productname>LZ4</productname>, if you want to support
compression of data with this method; see
- <xref linkend="guc-default-toast-compression"/>.
+ <xref linkend="guc-default-toast-compression"/> and
+ <xref linkend="guc-wal-compression"/>.
</para>
</listitem>
@@ -980,7 +981,7 @@ build-postgresql:
<para>
Build with <productname>LZ4</productname> compression support.
This allows the use of <productname>LZ4</productname> for
- compression of table data.
+ compression of table and WAL data.
</para>
</listitem>
</varlistentry>
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 4f81f19c49..a8794a941a 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
#define ZLIB_MAX_BLCKSZ 0
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ 0
+#endif
+
/* Buffer size required to store a compressed version of backup block image */
#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
-#define COMPRESS_BUFSIZE Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
/*
* For each block reference registered with XLogRegisterBuffer, we fill in
@@ -889,6 +896,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+ if (len == 0)
+ len = -1;
+ break;
+#endif
+
default:
/*
* It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 822b2612cd..753b38e58a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -38,6 +38,10 @@
#include <zlib.h>
#endif
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -72,6 +76,9 @@ const struct config_enum_entry wal_compression_options[] = {
{"1", WAL_COMPRESSION_PGLZ, true},
#ifdef HAVE_LIBZ
{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef USE_LZ4
+ {"lz4", WAL_COMPRESSION_LZ4, false},
#endif
{NULL, 0, false}
};
@@ -1622,6 +1629,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
}
#endif
+#ifdef USE_LZ4
+ case WAL_COMPRESSION_LZ4:
+ decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+ bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+ break;
+#endif
+
default:
report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
(uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 3991d35afd..dbf3155502 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -218,7 +218,7 @@
#full_page_writes = on # recover from partial page writes
#wal_log_hints = off # also do full page writes of non-critical updates
# (change requires restart)
-#wal_compression = off # enable compression of full-page writes: off, pglz, zlib
+#wal_compression = off # enable compression of full-page writes: off, pglz, zlib, lz4
#wal_init_zero = on # zero-fill new WAL files
#wal_recycle = on # recycle WAL files
#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 8b740af66d..0287592cd4 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -330,6 +330,7 @@ typedef enum WalCompression
WAL_COMPRESSION_NONE,
WAL_COMPRESSION_PGLZ,
WAL_COMPRESSION_ZLIB,
+ WAL_COMPRESSION_LZ4,
} WalCompression;
extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h
index 2a60c0fb92..abb42b364d 100644
--- a/src/include/access/xlogrecord.h
+++ b/src/include/access/xlogrecord.h
@@ -147,7 +147,7 @@ typedef struct XLogRecordBlockImageHeader
#define BKPIMAGE_APPLY 0x02 /* page image should be restored during
* replay */
#define BKPIMAGE_COMPRESS_METHOD1 0x04 /* bits to encode compression method */
-#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib */
+#define BKPIMAGE_COMPRESS_METHOD2 0x08 /* 0=none, 1=pglz, 2=zlib, 3=lz4 */
/* How many bits to shift to extract compression */
#define BKPIMAGE_COMPRESS_OFFSET_BITS 2
--
2.17.0
--P+33d92oIH25kiaB
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v8-0006-Default-to-LZ4.patch"
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
@ 2022-04-01 14:24 Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Nathan Bossart @ 2022-04-01 14:24 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]; [email protected]
On Fri, Apr 01, 2022 at 10:31:10AM +0900, Kyotaro Horiguchi wrote:
> you should at least save the contents of the cluster's <filename>pg_wal</filename>
> - subdirectory, as it might contain logs which
> + subdirectory, as it might contain WAL files which
> were not archived before the system went down.
>
> The "logs" means acutally "WAL segment (files)" but the concept of
> "segment" is out of focus in the context. So just "file" is used
> there. The same change is applied on dezon of places.
This change seems reasonable to me.
> - disk-space requirements for the <acronym>WAL</acronym> logs are met,
> + disk-space requirements for the <acronym>WAL</acronym> are met,
>
> This might be better be "WAL files" instead of just "WAL".
+1 for "WAL files"
> - <acronym>WAL</acronym> logs are stored in the directory
> + <acronym>WAL</acronym> is stored in the directory
> <filename>pg_wal</filename> under the data directory, as a set of
>
> I'm not sure which is better, use "WAL" as a collective noun, or "WAL
> files" as the cocrete objects.
My vote is for "WAL files" because it was previously "WAL logs."
> - The aim of <acronym>WAL</acronym> is to ensure that the log is
> + The aim of <acronym>WAL</acronym> is to ensure that the WAL record is
> written before database records are altered, but this can be subverted by
>
> This is not a mechanical change. But I think this is correct.
IMO the original wording is fine. I think it is sufficiently clear that
"log" refers to "write-ahead log," and this sentence seems intended to
convey the basic rule of "log before data." However, the rest of the
sentence is a little weird. It's basically saying "the aim of the log is
to ensure that the log is written..." Isn't the aim of the log to record
the database activity? Perhaps we should rewrite it to something like the
following:
A basic rule of WAL is that the log must be written before the database
files are altered, but this can be...
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
@ 2022-04-25 19:54 ` Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Nathan Bossart @ 2022-04-25 19:54 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: [email protected]; [email protected]
It's been a few weeks, so I'm marking the commitfest entry as
waiting-on-author.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
@ 2022-07-18 10:54 ` Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Bharath Rupireddy @ 2022-07-18 10:54 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Apr 26, 2022 at 1:24 AM Nathan Bossart <[email protected]> wrote:
>
> It's been a few weeks, so I'm marking the commitfest entry as
> waiting-on-author.
Thanks. I'm attaching the updated v4 patches (also subsumed Kyotaro
San's patch at [1]). Please review it further.
[1] https://www.postgresql.org/message-id/20220401.103110.1103213854487561781.horikyota.ntt%40gmail.com
Regards,
Bharath Rupireddy.
Attachments:
[application/octet-stream] v4-0001-Use-WAL-segment-instead-of-log-segment.patch (9.0K, ../../CALj2ACUxV-_jzwuYMZXbT=dJHuO2HNUzE67LorhqkSziqNOn+w@mail.gmail.com/2-v4-0001-Use-WAL-segment-instead-of-log-segment.patch)
download | inline diff:
From 68d39c9b087e25fc39312d72cca0ed31c1d9fad2 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 18 Jul 2022 10:27:20 +0000
Subject: [PATCH v4] Use "WAL segment" instead of "log segment"
We are using "log segment" in various user-facing messages, the
term "log" can mean server logs as well. The "WAL segment" suits
well here and it is consistently used across the other user-facing
messages.
Author: Bharath Rupireddy
---
src/backend/access/transam/xlogreader.c | 10 +++++-----
src/backend/access/transam/xlogrecovery.c | 6 +++---
src/backend/access/transam/xlogutils.c | 4 ++--
src/backend/replication/walreceiver.c | 6 +++---
src/bin/pg_resetwal/pg_resetwal.c | 2 +-
src/bin/pg_upgrade/controldata.c | 2 +-
src/bin/pg_waldump/pg_waldump.c | 4 ++--
7 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f3dc4b7797..58f1a32b00 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1209,7 +1209,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid magic number %04X in log segment %s, offset %u",
+ "invalid magic number %04X in WAL segment %s, offset %u",
hdr->xlp_magic,
fname,
offset);
@@ -1223,7 +1223,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1264,7 +1264,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
/* hmm, first page of file doesn't have a long header? */
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1283,7 +1283,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "unexpected pageaddr %X/%X in log segment %s, offset %u",
+ "unexpected pageaddr %X/%X in WAL segment %s, offset %u",
LSN_FORMAT_ARGS(hdr->xlp_pageaddr),
fname,
offset);
@@ -1308,7 +1308,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u",
+ "out-of-sequence timeline ID %u (after %u) in WAL segment %s, offset %u",
hdr->xlp_tli,
state->latestPageTLI,
fname,
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..306a9f40e9 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -3018,7 +3018,7 @@ ReadRecord(XLogPrefetcher *xlogprefetcher, int emode,
XLogFileName(fname, xlogreader->seg.ws_tli, segno,
wal_segment_size);
ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
- (errmsg("unexpected timeline ID %u in log segment %s, offset %u",
+ (errmsg("unexpected timeline ID %u in WAL segment %s, offset %u",
xlogreader->latestPageTLI,
fname,
offset)));
@@ -3223,13 +3223,13 @@ retry:
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %u: %m",
+ errmsg("could not read from WAL segment %s, offset %u: %m",
fname, readOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %u: read %d of %zu",
+ errmsg("could not read from WAL segment %s, offset %u: read %d of %zu",
fname, readOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 0cda22597f..9e3a000768 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1049,14 +1049,14 @@ WALReadRaiseError(WALReadError *errinfo)
errno = errinfo->wre_errno;
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %d: %m",
+ errmsg("could not read from WAL segment %s, offset %d: %m",
fname, errinfo->wre_off)));
}
else if (errinfo->wre_read == 0)
{
ereport(ERROR,
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %d: read %d of %d",
+ errmsg("could not read from WAL segment %s, offset %d: read %d of %d",
fname, errinfo->wre_off, errinfo->wre_read,
errinfo->wre_req)));
}
diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c
index 3d37c1fe62..3767466ef3 100644
--- a/src/backend/replication/walreceiver.c
+++ b/src/backend/replication/walreceiver.c
@@ -616,7 +616,7 @@ WalReceiverMain(void)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
@@ -930,7 +930,7 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr, TimeLineID tli)
errno = save_errno;
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not write to log segment %s "
+ errmsg("could not write to WAL segment %s "
"at offset %u, length %lu: %m",
xlogfname, startoff, (unsigned long) segbytes)));
}
@@ -1042,7 +1042,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index d4772a2965..7adf79eeed 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -788,7 +788,7 @@ PrintNewControlValues(void)
XLogFileName(fname, ControlFile.checkPointCopy.ThisTimeLineID,
newXlogSegNo, WalSegSz);
- printf(_("First log segment after reset: %s\n"), fname);
+ printf(_("First WAL segment after reset: %s\n"), fname);
if (set_mxid != 0)
{
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 07de918358..678e8ebf6b 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -350,7 +350,7 @@ get_control_data(ClusterInfo *cluster, bool live_check)
cluster->controldata.chkpnt_nxtmxoff = str2uint(p);
got_mxoff = true;
}
- else if ((p = strstr(bufin, "First log segment after reset:")) != NULL)
+ else if ((p = strstr(bufin, "First WAL segment after reset:")) != NULL)
{
/* Skip the colon and any whitespace after it */
p = strchr(p, ':');
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6528113628..4eebeadc8c 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -667,7 +667,7 @@ usage(void)
printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n"
" valid names are main, fsm, vm, init\n"));
printf(_(" -n, --limit=N number of records to display\n"));
- printf(_(" -p, --path=PATH directory in which to find log segment files or a\n"
+ printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n"
" directory with a ./pg_wal that contains such files\n"
" (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n"));
printf(_(" -q, --quiet do not print any output, except for errors\n"));
@@ -675,7 +675,7 @@ usage(void)
" use --rmgr=list to list valid resource manager names\n"));
printf(_(" -R, --relation=T/D/R only show records that modify blocks in relation T/D/R\n"));
printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n"));
- printf(_(" -t, --timeline=TLI timeline from which to read log records\n"
+ printf(_(" -t, --timeline=TLI timeline from which to read WAL records\n"
" (default: 1 or the value used in STARTSEG)\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -w, --fullpage only show records with a full page write\n"));
--
2.25.1
[application/octet-stream] v4-0002-Replace-log-record-with-WAL-record-in-docs.patch (17.3K, ../../CALj2ACUxV-_jzwuYMZXbT=dJHuO2HNUzE67LorhqkSziqNOn+w@mail.gmail.com/3-v4-0002-Replace-log-record-with-WAL-record-in-docs.patch)
download | inline diff:
From 654984e6ee83de08a800ac4f49a67b9c2d13fe27 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 18 Jul 2022 10:48:52 +0000
Subject: [PATCH v4] Replace log record with WAL record in docs
Authors: Kyotaro Horiguchi, Bharath Rupireddy
---
doc/src/sgml/backup.sgml | 14 ++++----
doc/src/sgml/ref/pg_waldump.sgml | 10 +++---
doc/src/sgml/wal.sgml | 60 ++++++++++++++++----------------
3 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 73a774d3d7..cc5ae59ac2 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1095,7 +1095,7 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
require that you have enough free space on your system to hold two
copies of your existing database. If you do not have enough space,
you should at least save the contents of the cluster's <filename>pg_wal</filename>
- subdirectory, as it might contain logs which
+ subdirectory, as it might contain WAL files which
were not archived before the system went down.
</para>
</listitem>
@@ -1173,8 +1173,8 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
which tells <productname>PostgreSQL</productname> how to retrieve archived
WAL file segments. Like the <varname>archive_command</varname>, this is
a shell command string. It can contain <literal>%f</literal>, which is
- replaced by the name of the desired log file, and <literal>%p</literal>,
- which is replaced by the path name to copy the log file to.
+ replaced by the name of the desired WAL file, and <literal>%p</literal>,
+ which is replaced by the path name to copy the WAL file to.
(The path name is relative to the current working directory,
i.e., the cluster's data directory.)
Write <literal>%%</literal> if you need to embed an actual <literal>%</literal>
@@ -1462,9 +1462,9 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
<link linkend="sql-createtablespace"><command>CREATE TABLESPACE</command></link>
commands are WAL-logged with the literal absolute path, and will
therefore be replayed as tablespace creations with the same
- absolute path. This might be undesirable if the log is being
+ absolute path. This might be undesirable if the WAL is being
replayed on a different machine. It can be dangerous even if the
- log is being replayed on the same machine, but into a new data
+ WAL is being replayed on the same machine, but into a new data
directory: the replay will still overwrite the contents of the
original tablespace. To avoid potential gotchas of this sort,
the best practice is to take a new base backup after creating or
@@ -1481,11 +1481,11 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
we might need to fix partially-written disk pages. Depending on
your system hardware and software, the risk of partial writes might
be small enough to ignore, in which case you can significantly
- reduce the total volume of archived logs by turning off page
+ reduce the total volume of archived WAL files by turning off page
snapshots using the <xref linkend="guc-full-page-writes"/>
parameter. (Read the notes and warnings in <xref linkend="wal"/>
before you do so.) Turning off page snapshots does not prevent
- use of the logs for PITR operations. An area for future
+ use of the WAL for PITR operations. An area for future
development is to compress archived WAL data by removing
unnecessary page copies even when <varname>full_page_writes</varname> is
on. In the meantime, administrators might wish to reduce the number
diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml
index 57746d9421..2e2166bb6f 100644
--- a/doc/src/sgml/ref/pg_waldump.sgml
+++ b/doc/src/sgml/ref/pg_waldump.sgml
@@ -53,7 +53,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">startseg</replaceable></term>
<listitem>
<para>
- Start reading at the specified log segment file. This implicitly determines
+ Start reading at the specified WAL segment file. This implicitly determines
the path in which files will be searched for, and the timeline to use.
</para>
</listitem>
@@ -63,7 +63,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">endseg</replaceable></term>
<listitem>
<para>
- Stop after reading the specified log segment file.
+ Stop after reading the specified WAL segment file.
</para>
</listitem>
</varlistentry>
@@ -141,7 +141,7 @@ PostgreSQL documentation
<term><option>--path=<replaceable>path</replaceable></option></term>
<listitem>
<para>
- Specifies a directory to search for log segment files or a
+ Specifies a directory to search for WAL segment files or a
directory with a <literal>pg_wal</literal> subdirectory that
contains such files. The default is to search in the current
directory, the <literal>pg_wal</literal> subdirectory of the
@@ -203,7 +203,7 @@ PostgreSQL documentation
<listitem>
<para>
WAL location at which to start reading. The default is to start reading
- the first valid log record found in the earliest file found.
+ the first valid WAL record found in the earliest file found.
</para>
</listitem>
</varlistentry>
@@ -213,7 +213,7 @@ PostgreSQL documentation
<term><option>--timeline=<replaceable>timeline</replaceable></option></term>
<listitem>
<para>
- Timeline from which to read log records. The default is to use the
+ Timeline from which to read WAL records. The default is to use the
value in <replaceable>startseg</replaceable>, if that is specified; otherwise, the
default is 1.
</para>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 4b6ef283c1..e8bd7ffecf 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -296,12 +296,12 @@
transaction processing. Briefly, <acronym>WAL</acronym>'s central
concept is that changes to data files (where tables and indexes
reside) must be written only after those changes have been logged,
- that is, after log records describing the changes have been flushed
+ that is, after WAL records describing the changes have been flushed
to permanent storage. If we follow this procedure, we do not need
to flush data pages to disk on every transaction commit, because we
know that in the event of a crash we will be able to recover the
database using the log: any changes that have not been applied to
- the data pages can be redone from the log records. (This is
+ the data pages can be redone from the WAL records. (This is
roll-forward recovery, also known as REDO.)
</para>
@@ -322,15 +322,15 @@
<para>
Using <acronym>WAL</acronym> results in a
- significantly reduced number of disk writes, because only the log
+ significantly reduced number of disk writes, because only the WAL
file needs to be flushed to disk to guarantee that a transaction is
committed, rather than every data file changed by the transaction.
- The log file is written sequentially,
- and so the cost of syncing the log is much less than the cost of
+ The WAL file is written sequentially,
+ and so the cost of syncing the WAL is much less than the cost of
flushing the data pages. This is especially true for servers
handling many small transactions touching different parts of the data
store. Furthermore, when the server is processing many small concurrent
- transactions, one <function>fsync</function> of the log file may
+ transactions, one <function>fsync</function> of the WAL file may
suffice to commit many transactions.
</para>
@@ -340,10 +340,10 @@
linkend="continuous-archiving"/>. By archiving the WAL data we can support
reverting to any time instant covered by the available WAL data:
we simply install a prior physical backup of the database, and
- replay the WAL log just as far as the desired time. What's more,
+ replay the WAL just as far as the desired time. What's more,
the physical backup doesn't have to be an instantaneous snapshot
of the database state — if it is made over some period of time,
- then replaying the WAL log for that period will fix any internal
+ then replaying the WAL for that period will fix any internal
inconsistencies.
</para>
</sect1>
@@ -496,15 +496,15 @@
that the heap and index data files have been updated with all
information written before that checkpoint. At checkpoint time, all
dirty data pages are flushed to disk and a special checkpoint record is
- written to the log file. (The change records were previously flushed
+ written to the WAL file. (The change records were previously flushed
to the <acronym>WAL</acronym> files.)
In the event of a crash, the crash recovery procedure looks at the latest
- checkpoint record to determine the point in the log (known as the redo
+ checkpoint record to determine the point in the WAL (known as the redo
record) from which it should start the REDO operation. Any changes made to
data files before that point are guaranteed to be already on disk.
- Hence, after a checkpoint, log segments preceding the one containing
+ Hence, after a checkpoint, WAL segments preceding the one containing
the redo record are no longer needed and can be recycled or removed. (When
- <acronym>WAL</acronym> archiving is being done, the log segments must be
+ <acronym>WAL</acronym> archiving is being done, the WAL segments must be
archived before being recycled or removed.)
</para>
@@ -543,7 +543,7 @@
another factor to consider. To ensure data page consistency,
the first modification of a data page after each checkpoint results in
logging the entire page content. In that case,
- a smaller checkpoint interval increases the volume of output to the WAL log,
+ a smaller checkpoint interval increases the volume of output to the WAL,
partially negating the goal of using a smaller interval,
and in any case causing more disk I/O.
</para>
@@ -613,10 +613,10 @@
<para>
The number of WAL segment files in <filename>pg_wal</filename> directory depends on
<varname>min_wal_size</varname>, <varname>max_wal_size</varname> and
- the amount of WAL generated in previous checkpoint cycles. When old log
+ the amount of WAL generated in previous checkpoint cycles. When old WAL
segment files are no longer needed, they are removed or recycled (that is,
renamed to become future segments in the numbered sequence). If, due to a
- short-term peak of log output rate, <varname>max_wal_size</varname> is
+ short-term peak of WAL output rate, <varname>max_wal_size</varname> is
exceeded, the unneeded segment files will be removed until the system
gets back under this limit. Below that limit, the system recycles enough
WAL files to cover the estimated need until the next checkpoint, and
@@ -649,7 +649,7 @@
which are similar to checkpoints in normal operation: the server forces
all its state to disk, updates the <filename>pg_control</filename> file to
indicate that the already-processed WAL data need not be scanned again,
- and then recycles any old log segment files in the <filename>pg_wal</filename>
+ and then recycles any old WAL segment files in the <filename>pg_wal</filename>
directory.
Restartpoints can't be performed more frequently than checkpoints on the
primary because restartpoints can only be performed at checkpoint records.
@@ -675,12 +675,12 @@
insertion) at a time when an exclusive lock is held on affected
data pages, so the operation needs to be as fast as possible. What
is worse, writing <acronym>WAL</acronym> buffers might also force the
- creation of a new log segment, which takes even more
+ creation of a new WAL segment, which takes even more
time. Normally, <acronym>WAL</acronym> buffers should be written
and flushed by an <function>XLogFlush</function> request, which is
made, for the most part, at transaction commit time to ensure that
transaction records are flushed to permanent storage. On systems
- with high log output, <function>XLogFlush</function> requests might
+ with high WAL output, <function>XLogFlush</function> requests might
not occur often enough to prevent <function>XLogInsertRecord</function>
from having to do writes. On such systems
one should increase the number of <acronym>WAL</acronym> buffers by
@@ -723,7 +723,7 @@
<varname>commit_delay</varname>, so this value is recommended as the
starting point to use when optimizing for a particular workload. While
tuning <varname>commit_delay</varname> is particularly useful when the
- WAL log is stored on high-latency rotating disks, benefits can be
+ WAL is stored on high-latency rotating disks, benefits can be
significant even on storage media with very fast sync times, such as
solid-state drives or RAID arrays with a battery-backed write cache;
but this should definitely be tested against a representative workload.
@@ -827,16 +827,16 @@
<para>
<acronym>WAL</acronym> is automatically enabled; no action is
required from the administrator except ensuring that the
- disk-space requirements for the <acronym>WAL</acronym> logs are met,
+ disk-space requirements for the <acronym>WAL</acronym> files are met,
and that any necessary tuning is done (see <xref
linkend="wal-configuration"/>).
</para>
<para>
<acronym>WAL</acronym> records are appended to the <acronym>WAL</acronym>
- logs as each new record is written. The insert position is described by
+ files as each new record is written. The insert position is described by
a Log Sequence Number (<acronym>LSN</acronym>) that is a byte offset into
- the logs, increasing monotonically with each new record.
+ the WAL, increasing monotonically with each new record.
<acronym>LSN</acronym> values are returned as the datatype
<link linkend="datatype-pg-lsn"><type>pg_lsn</type></link>. Values can be
compared to calculate the volume of <acronym>WAL</acronym> data that
@@ -845,12 +845,12 @@
</para>
<para>
- <acronym>WAL</acronym> logs are stored in the directory
+ <acronym>WAL</acronym> files are stored in the directory
<filename>pg_wal</filename> under the data directory, as a set of
segment files, normally each 16 MB in size (but the size can be changed
by altering the <option>--wal-segsize</option> <application>initdb</application> option). Each segment is
divided into pages, normally 8 kB each (this size can be changed via the
- <option>--with-wal-blocksize</option> configure option). The log record headers
+ <option>--with-wal-blocksize</option> configure option). The WAL record headers
are described in <filename>access/xlogrecord.h</filename>; the record
content is dependent on the type of event that is being logged. Segment
files are given ever-increasing numbers as names, starting at
@@ -860,7 +860,7 @@
</para>
<para>
- It is advantageous if the log is located on a different disk from the
+ It is advantageous if the WAL is located on a different disk from the
main database files. This can be achieved by moving the
<filename>pg_wal</filename> directory to another location (while the server
is shut down, of course) and creating a symbolic link from the
@@ -876,19 +876,19 @@
on the disk. A power failure in such a situation might lead to
irrecoverable data corruption. Administrators should try to ensure
that disks holding <productname>PostgreSQL</productname>'s
- <acronym>WAL</acronym> log files do not make such false reports.
+ <acronym>WAL</acronym> files do not make such false reports.
(See <xref linkend="wal-reliability"/>.)
</para>
<para>
- After a checkpoint has been made and the log flushed, the
+ After a checkpoint has been made and the WAL flushed, the
checkpoint's position is saved in the file
<filename>pg_control</filename>. Therefore, at the start of recovery,
the server first reads <filename>pg_control</filename> and
then the checkpoint record; then it performs the REDO operation by
- scanning forward from the log location indicated in the checkpoint
+ scanning forward from the WAL location indicated in the checkpoint
record. Because the entire content of data pages is saved in the
- log on the first page modification after a checkpoint (assuming
+ WAL record on the first page modification after a checkpoint (assuming
<xref linkend="guc-full-page-writes"/> is not disabled), all pages
changed since the checkpoint will be restored to a consistent
state.
@@ -896,7 +896,7 @@
<para>
To deal with the case where <filename>pg_control</filename> is
- corrupt, we should support the possibility of scanning existing log
+ corrupt, we should support the possibility of scanning existing WAL
segments in reverse order — newest to oldest — in order to find the
latest checkpoint. This has not been implemented yet.
<filename>pg_control</filename> is small enough (less than one disk page)
--
2.25.1
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
@ 2022-07-18 18:30 ` Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Nathan Bossart @ 2022-07-18 18:30 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; PostgreSQL Hackers <[email protected]>
Overall, these patches look reasonable.
On Mon, Jul 18, 2022 at 04:24:12PM +0530, Bharath Rupireddy wrote:
> record. Because the entire content of data pages is saved in the
> - log on the first page modification after a checkpoint (assuming
> + WAL record on the first page modification after a checkpoint (assuming
> <xref linkend="guc-full-page-writes"/> is not disabled), all pages
> changed since the checkpoint will be restored to a consistent
> state.
nitpick: I would remove the word "record" in this change.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
@ 2022-07-19 09:13 ` Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Bharath Rupireddy @ 2022-07-19 09:13 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Jul 19, 2022 at 12:00 AM Nathan Bossart
<[email protected]> wrote:
>
> Overall, these patches look reasonable.
>
> On Mon, Jul 18, 2022 at 04:24:12PM +0530, Bharath Rupireddy wrote:
> > record. Because the entire content of data pages is saved in the
> > - log on the first page modification after a checkpoint (assuming
> > + WAL record on the first page modification after a checkpoint (assuming
> > <xref linkend="guc-full-page-writes"/> is not disabled), all pages
> > changed since the checkpoint will be restored to a consistent
> > state.
>
> nitpick: I would remove the word "record" in this change.
Done. PSA v5 patch set.
Regards,
Bharath Rupireddy.
Attachments:
[application/octet-stream] v5-0001-Use-WAL-segment-instead-of-log-segment.patch (9.0K, ../../CALj2ACVTtH9HamzzYTSFuEopeh77homm3mW2fx13QPnsnGAKUg@mail.gmail.com/2-v5-0001-Use-WAL-segment-instead-of-log-segment.patch)
download | inline diff:
From 68d39c9b087e25fc39312d72cca0ed31c1d9fad2 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Mon, 18 Jul 2022 10:27:20 +0000
Subject: [PATCH v5] Use "WAL segment" instead of "log segment"
We are using "log segment" in various user-facing messages, the
term "log" can mean server logs as well. The "WAL segment" suits
well here and it is consistently used across the other user-facing
messages.
Author: Bharath Rupireddy
---
src/backend/access/transam/xlogreader.c | 10 +++++-----
src/backend/access/transam/xlogrecovery.c | 6 +++---
src/backend/access/transam/xlogutils.c | 4 ++--
src/backend/replication/walreceiver.c | 6 +++---
src/bin/pg_resetwal/pg_resetwal.c | 2 +-
src/bin/pg_upgrade/controldata.c | 2 +-
src/bin/pg_waldump/pg_waldump.c | 4 ++--
7 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f3dc4b7797..58f1a32b00 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1209,7 +1209,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid magic number %04X in log segment %s, offset %u",
+ "invalid magic number %04X in WAL segment %s, offset %u",
hdr->xlp_magic,
fname,
offset);
@@ -1223,7 +1223,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1264,7 +1264,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
/* hmm, first page of file doesn't have a long header? */
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1283,7 +1283,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "unexpected pageaddr %X/%X in log segment %s, offset %u",
+ "unexpected pageaddr %X/%X in WAL segment %s, offset %u",
LSN_FORMAT_ARGS(hdr->xlp_pageaddr),
fname,
offset);
@@ -1308,7 +1308,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u",
+ "out-of-sequence timeline ID %u (after %u) in WAL segment %s, offset %u",
hdr->xlp_tli,
state->latestPageTLI,
fname,
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..306a9f40e9 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -3018,7 +3018,7 @@ ReadRecord(XLogPrefetcher *xlogprefetcher, int emode,
XLogFileName(fname, xlogreader->seg.ws_tli, segno,
wal_segment_size);
ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
- (errmsg("unexpected timeline ID %u in log segment %s, offset %u",
+ (errmsg("unexpected timeline ID %u in WAL segment %s, offset %u",
xlogreader->latestPageTLI,
fname,
offset)));
@@ -3223,13 +3223,13 @@ retry:
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %u: %m",
+ errmsg("could not read from WAL segment %s, offset %u: %m",
fname, readOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %u: read %d of %zu",
+ errmsg("could not read from WAL segment %s, offset %u: read %d of %zu",
fname, readOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 0cda22597f..9e3a000768 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1049,14 +1049,14 @@ WALReadRaiseError(WALReadError *errinfo)
errno = errinfo->wre_errno;
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %d: %m",
+ errmsg("could not read from WAL segment %s, offset %d: %m",
fname, errinfo->wre_off)));
}
else if (errinfo->wre_read == 0)
{
ereport(ERROR,
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %d: read %d of %d",
+ errmsg("could not read from WAL segment %s, offset %d: read %d of %d",
fname, errinfo->wre_off, errinfo->wre_read,
errinfo->wre_req)));
}
diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c
index 3d37c1fe62..3767466ef3 100644
--- a/src/backend/replication/walreceiver.c
+++ b/src/backend/replication/walreceiver.c
@@ -616,7 +616,7 @@ WalReceiverMain(void)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
@@ -930,7 +930,7 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr, TimeLineID tli)
errno = save_errno;
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not write to log segment %s "
+ errmsg("could not write to WAL segment %s "
"at offset %u, length %lu: %m",
xlogfname, startoff, (unsigned long) segbytes)));
}
@@ -1042,7 +1042,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index d4772a2965..7adf79eeed 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -788,7 +788,7 @@ PrintNewControlValues(void)
XLogFileName(fname, ControlFile.checkPointCopy.ThisTimeLineID,
newXlogSegNo, WalSegSz);
- printf(_("First log segment after reset: %s\n"), fname);
+ printf(_("First WAL segment after reset: %s\n"), fname);
if (set_mxid != 0)
{
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 07de918358..678e8ebf6b 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -350,7 +350,7 @@ get_control_data(ClusterInfo *cluster, bool live_check)
cluster->controldata.chkpnt_nxtmxoff = str2uint(p);
got_mxoff = true;
}
- else if ((p = strstr(bufin, "First log segment after reset:")) != NULL)
+ else if ((p = strstr(bufin, "First WAL segment after reset:")) != NULL)
{
/* Skip the colon and any whitespace after it */
p = strchr(p, ':');
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6528113628..4eebeadc8c 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -667,7 +667,7 @@ usage(void)
printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n"
" valid names are main, fsm, vm, init\n"));
printf(_(" -n, --limit=N number of records to display\n"));
- printf(_(" -p, --path=PATH directory in which to find log segment files or a\n"
+ printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n"
" directory with a ./pg_wal that contains such files\n"
" (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n"));
printf(_(" -q, --quiet do not print any output, except for errors\n"));
@@ -675,7 +675,7 @@ usage(void)
" use --rmgr=list to list valid resource manager names\n"));
printf(_(" -R, --relation=T/D/R only show records that modify blocks in relation T/D/R\n"));
printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n"));
- printf(_(" -t, --timeline=TLI timeline from which to read log records\n"
+ printf(_(" -t, --timeline=TLI timeline from which to read WAL records\n"
" (default: 1 or the value used in STARTSEG)\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -w, --fullpage only show records with a full page write\n"));
--
2.25.1
[application/octet-stream] v5-0002-Replace-log-record-with-WAL-record-in-docs.patch (17.2K, ../../CALj2ACVTtH9HamzzYTSFuEopeh77homm3mW2fx13QPnsnGAKUg@mail.gmail.com/3-v5-0002-Replace-log-record-with-WAL-record-in-docs.patch)
download | inline diff:
From 8657cf222102cd6f15ee773c6301ed41be2831eb Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Tue, 19 Jul 2022 09:12:12 +0000
Subject: [PATCH v5] Replace log record with WAL record in docs
Authors: Kyotaro Horiguchi, Bharath Rupireddy
---
doc/src/sgml/backup.sgml | 14 ++++----
doc/src/sgml/ref/pg_waldump.sgml | 10 +++---
doc/src/sgml/wal.sgml | 60 ++++++++++++++++----------------
3 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 73a774d3d7..cc5ae59ac2 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1095,7 +1095,7 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
require that you have enough free space on your system to hold two
copies of your existing database. If you do not have enough space,
you should at least save the contents of the cluster's <filename>pg_wal</filename>
- subdirectory, as it might contain logs which
+ subdirectory, as it might contain WAL files which
were not archived before the system went down.
</para>
</listitem>
@@ -1173,8 +1173,8 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
which tells <productname>PostgreSQL</productname> how to retrieve archived
WAL file segments. Like the <varname>archive_command</varname>, this is
a shell command string. It can contain <literal>%f</literal>, which is
- replaced by the name of the desired log file, and <literal>%p</literal>,
- which is replaced by the path name to copy the log file to.
+ replaced by the name of the desired WAL file, and <literal>%p</literal>,
+ which is replaced by the path name to copy the WAL file to.
(The path name is relative to the current working directory,
i.e., the cluster's data directory.)
Write <literal>%%</literal> if you need to embed an actual <literal>%</literal>
@@ -1462,9 +1462,9 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
<link linkend="sql-createtablespace"><command>CREATE TABLESPACE</command></link>
commands are WAL-logged with the literal absolute path, and will
therefore be replayed as tablespace creations with the same
- absolute path. This might be undesirable if the log is being
+ absolute path. This might be undesirable if the WAL is being
replayed on a different machine. It can be dangerous even if the
- log is being replayed on the same machine, but into a new data
+ WAL is being replayed on the same machine, but into a new data
directory: the replay will still overwrite the contents of the
original tablespace. To avoid potential gotchas of this sort,
the best practice is to take a new base backup after creating or
@@ -1481,11 +1481,11 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
we might need to fix partially-written disk pages. Depending on
your system hardware and software, the risk of partial writes might
be small enough to ignore, in which case you can significantly
- reduce the total volume of archived logs by turning off page
+ reduce the total volume of archived WAL files by turning off page
snapshots using the <xref linkend="guc-full-page-writes"/>
parameter. (Read the notes and warnings in <xref linkend="wal"/>
before you do so.) Turning off page snapshots does not prevent
- use of the logs for PITR operations. An area for future
+ use of the WAL for PITR operations. An area for future
development is to compress archived WAL data by removing
unnecessary page copies even when <varname>full_page_writes</varname> is
on. In the meantime, administrators might wish to reduce the number
diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml
index 57746d9421..2e2166bb6f 100644
--- a/doc/src/sgml/ref/pg_waldump.sgml
+++ b/doc/src/sgml/ref/pg_waldump.sgml
@@ -53,7 +53,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">startseg</replaceable></term>
<listitem>
<para>
- Start reading at the specified log segment file. This implicitly determines
+ Start reading at the specified WAL segment file. This implicitly determines
the path in which files will be searched for, and the timeline to use.
</para>
</listitem>
@@ -63,7 +63,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">endseg</replaceable></term>
<listitem>
<para>
- Stop after reading the specified log segment file.
+ Stop after reading the specified WAL segment file.
</para>
</listitem>
</varlistentry>
@@ -141,7 +141,7 @@ PostgreSQL documentation
<term><option>--path=<replaceable>path</replaceable></option></term>
<listitem>
<para>
- Specifies a directory to search for log segment files or a
+ Specifies a directory to search for WAL segment files or a
directory with a <literal>pg_wal</literal> subdirectory that
contains such files. The default is to search in the current
directory, the <literal>pg_wal</literal> subdirectory of the
@@ -203,7 +203,7 @@ PostgreSQL documentation
<listitem>
<para>
WAL location at which to start reading. The default is to start reading
- the first valid log record found in the earliest file found.
+ the first valid WAL record found in the earliest file found.
</para>
</listitem>
</varlistentry>
@@ -213,7 +213,7 @@ PostgreSQL documentation
<term><option>--timeline=<replaceable>timeline</replaceable></option></term>
<listitem>
<para>
- Timeline from which to read log records. The default is to use the
+ Timeline from which to read WAL records. The default is to use the
value in <replaceable>startseg</replaceable>, if that is specified; otherwise, the
default is 1.
</para>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 4b6ef283c1..315ce5f8bd 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -296,12 +296,12 @@
transaction processing. Briefly, <acronym>WAL</acronym>'s central
concept is that changes to data files (where tables and indexes
reside) must be written only after those changes have been logged,
- that is, after log records describing the changes have been flushed
+ that is, after WAL records describing the changes have been flushed
to permanent storage. If we follow this procedure, we do not need
to flush data pages to disk on every transaction commit, because we
know that in the event of a crash we will be able to recover the
database using the log: any changes that have not been applied to
- the data pages can be redone from the log records. (This is
+ the data pages can be redone from the WAL records. (This is
roll-forward recovery, also known as REDO.)
</para>
@@ -322,15 +322,15 @@
<para>
Using <acronym>WAL</acronym> results in a
- significantly reduced number of disk writes, because only the log
+ significantly reduced number of disk writes, because only the WAL
file needs to be flushed to disk to guarantee that a transaction is
committed, rather than every data file changed by the transaction.
- The log file is written sequentially,
- and so the cost of syncing the log is much less than the cost of
+ The WAL file is written sequentially,
+ and so the cost of syncing the WAL is much less than the cost of
flushing the data pages. This is especially true for servers
handling many small transactions touching different parts of the data
store. Furthermore, when the server is processing many small concurrent
- transactions, one <function>fsync</function> of the log file may
+ transactions, one <function>fsync</function> of the WAL file may
suffice to commit many transactions.
</para>
@@ -340,10 +340,10 @@
linkend="continuous-archiving"/>. By archiving the WAL data we can support
reverting to any time instant covered by the available WAL data:
we simply install a prior physical backup of the database, and
- replay the WAL log just as far as the desired time. What's more,
+ replay the WAL just as far as the desired time. What's more,
the physical backup doesn't have to be an instantaneous snapshot
of the database state — if it is made over some period of time,
- then replaying the WAL log for that period will fix any internal
+ then replaying the WAL for that period will fix any internal
inconsistencies.
</para>
</sect1>
@@ -496,15 +496,15 @@
that the heap and index data files have been updated with all
information written before that checkpoint. At checkpoint time, all
dirty data pages are flushed to disk and a special checkpoint record is
- written to the log file. (The change records were previously flushed
+ written to the WAL file. (The change records were previously flushed
to the <acronym>WAL</acronym> files.)
In the event of a crash, the crash recovery procedure looks at the latest
- checkpoint record to determine the point in the log (known as the redo
+ checkpoint record to determine the point in the WAL (known as the redo
record) from which it should start the REDO operation. Any changes made to
data files before that point are guaranteed to be already on disk.
- Hence, after a checkpoint, log segments preceding the one containing
+ Hence, after a checkpoint, WAL segments preceding the one containing
the redo record are no longer needed and can be recycled or removed. (When
- <acronym>WAL</acronym> archiving is being done, the log segments must be
+ <acronym>WAL</acronym> archiving is being done, the WAL segments must be
archived before being recycled or removed.)
</para>
@@ -543,7 +543,7 @@
another factor to consider. To ensure data page consistency,
the first modification of a data page after each checkpoint results in
logging the entire page content. In that case,
- a smaller checkpoint interval increases the volume of output to the WAL log,
+ a smaller checkpoint interval increases the volume of output to the WAL,
partially negating the goal of using a smaller interval,
and in any case causing more disk I/O.
</para>
@@ -613,10 +613,10 @@
<para>
The number of WAL segment files in <filename>pg_wal</filename> directory depends on
<varname>min_wal_size</varname>, <varname>max_wal_size</varname> and
- the amount of WAL generated in previous checkpoint cycles. When old log
+ the amount of WAL generated in previous checkpoint cycles. When old WAL
segment files are no longer needed, they are removed or recycled (that is,
renamed to become future segments in the numbered sequence). If, due to a
- short-term peak of log output rate, <varname>max_wal_size</varname> is
+ short-term peak of WAL output rate, <varname>max_wal_size</varname> is
exceeded, the unneeded segment files will be removed until the system
gets back under this limit. Below that limit, the system recycles enough
WAL files to cover the estimated need until the next checkpoint, and
@@ -649,7 +649,7 @@
which are similar to checkpoints in normal operation: the server forces
all its state to disk, updates the <filename>pg_control</filename> file to
indicate that the already-processed WAL data need not be scanned again,
- and then recycles any old log segment files in the <filename>pg_wal</filename>
+ and then recycles any old WAL segment files in the <filename>pg_wal</filename>
directory.
Restartpoints can't be performed more frequently than checkpoints on the
primary because restartpoints can only be performed at checkpoint records.
@@ -675,12 +675,12 @@
insertion) at a time when an exclusive lock is held on affected
data pages, so the operation needs to be as fast as possible. What
is worse, writing <acronym>WAL</acronym> buffers might also force the
- creation of a new log segment, which takes even more
+ creation of a new WAL segment, which takes even more
time. Normally, <acronym>WAL</acronym> buffers should be written
and flushed by an <function>XLogFlush</function> request, which is
made, for the most part, at transaction commit time to ensure that
transaction records are flushed to permanent storage. On systems
- with high log output, <function>XLogFlush</function> requests might
+ with high WAL output, <function>XLogFlush</function> requests might
not occur often enough to prevent <function>XLogInsertRecord</function>
from having to do writes. On such systems
one should increase the number of <acronym>WAL</acronym> buffers by
@@ -723,7 +723,7 @@
<varname>commit_delay</varname>, so this value is recommended as the
starting point to use when optimizing for a particular workload. While
tuning <varname>commit_delay</varname> is particularly useful when the
- WAL log is stored on high-latency rotating disks, benefits can be
+ WAL is stored on high-latency rotating disks, benefits can be
significant even on storage media with very fast sync times, such as
solid-state drives or RAID arrays with a battery-backed write cache;
but this should definitely be tested against a representative workload.
@@ -827,16 +827,16 @@
<para>
<acronym>WAL</acronym> is automatically enabled; no action is
required from the administrator except ensuring that the
- disk-space requirements for the <acronym>WAL</acronym> logs are met,
+ disk-space requirements for the <acronym>WAL</acronym> files are met,
and that any necessary tuning is done (see <xref
linkend="wal-configuration"/>).
</para>
<para>
<acronym>WAL</acronym> records are appended to the <acronym>WAL</acronym>
- logs as each new record is written. The insert position is described by
+ files as each new record is written. The insert position is described by
a Log Sequence Number (<acronym>LSN</acronym>) that is a byte offset into
- the logs, increasing monotonically with each new record.
+ the WAL, increasing monotonically with each new record.
<acronym>LSN</acronym> values are returned as the datatype
<link linkend="datatype-pg-lsn"><type>pg_lsn</type></link>. Values can be
compared to calculate the volume of <acronym>WAL</acronym> data that
@@ -845,12 +845,12 @@
</para>
<para>
- <acronym>WAL</acronym> logs are stored in the directory
+ <acronym>WAL</acronym> files are stored in the directory
<filename>pg_wal</filename> under the data directory, as a set of
segment files, normally each 16 MB in size (but the size can be changed
by altering the <option>--wal-segsize</option> <application>initdb</application> option). Each segment is
divided into pages, normally 8 kB each (this size can be changed via the
- <option>--with-wal-blocksize</option> configure option). The log record headers
+ <option>--with-wal-blocksize</option> configure option). The WAL record headers
are described in <filename>access/xlogrecord.h</filename>; the record
content is dependent on the type of event that is being logged. Segment
files are given ever-increasing numbers as names, starting at
@@ -860,7 +860,7 @@
</para>
<para>
- It is advantageous if the log is located on a different disk from the
+ It is advantageous if the WAL is located on a different disk from the
main database files. This can be achieved by moving the
<filename>pg_wal</filename> directory to another location (while the server
is shut down, of course) and creating a symbolic link from the
@@ -876,19 +876,19 @@
on the disk. A power failure in such a situation might lead to
irrecoverable data corruption. Administrators should try to ensure
that disks holding <productname>PostgreSQL</productname>'s
- <acronym>WAL</acronym> log files do not make such false reports.
+ <acronym>WAL</acronym> files do not make such false reports.
(See <xref linkend="wal-reliability"/>.)
</para>
<para>
- After a checkpoint has been made and the log flushed, the
+ After a checkpoint has been made and the WAL flushed, the
checkpoint's position is saved in the file
<filename>pg_control</filename>. Therefore, at the start of recovery,
the server first reads <filename>pg_control</filename> and
then the checkpoint record; then it performs the REDO operation by
- scanning forward from the log location indicated in the checkpoint
+ scanning forward from the WAL location indicated in the checkpoint
record. Because the entire content of data pages is saved in the
- log on the first page modification after a checkpoint (assuming
+ WAL on the first page modification after a checkpoint (assuming
<xref linkend="guc-full-page-writes"/> is not disabled), all pages
changed since the checkpoint will be restored to a consistent
state.
@@ -896,7 +896,7 @@
<para>
To deal with the case where <filename>pg_control</filename> is
- corrupt, we should support the possibility of scanning existing log
+ corrupt, we should support the possibility of scanning existing WAL
segments in reverse order — newest to oldest — in order to find the
latest checkpoint. This has not been implemented yet.
<filename>pg_control</filename> is small enough (less than one disk page)
--
2.25.1
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
@ 2022-07-19 16:58 ` Nathan Bossart <[email protected]>
2022-07-20 01:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Nathan Bossart @ 2022-07-19 16:58 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Kyotaro Horiguchi <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, Jul 19, 2022 at 02:43:59PM +0530, Bharath Rupireddy wrote:
> Done. PSA v5 patch set.
LGTM. I've marked this as ready-for-committer.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
@ 2022-07-20 01:25 ` Kyotaro Horiguchi <[email protected]>
2022-07-20 04:32 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Kyotaro Horiguchi @ 2022-07-20 01:25 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]
At Tue, 19 Jul 2022 09:58:28 -0700, Nathan Bossart <[email protected]> wrote in
> On Tue, Jul 19, 2022 at 02:43:59PM +0530, Bharath Rupireddy wrote:
> > Done. PSA v5 patch set.
>
> LGTM. I've marked this as ready-for-committer.
I find the following sentense in config.sgml. "Specifies the minimum
size of past log file segments kept in the pg_wal directory"
postgresql.conf.sample contains "logfile segment" in a few lines.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-20 01:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
@ 2022-07-20 04:32 ` Bharath Rupireddy <[email protected]>
2022-07-20 07:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Bharath Rupireddy @ 2022-07-20 04:32 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: Nathan Bossart <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jul 20, 2022 at 6:55 AM Kyotaro Horiguchi
<[email protected]> wrote:
>
> At Tue, 19 Jul 2022 09:58:28 -0700, Nathan Bossart <[email protected]> wrote in
> > On Tue, Jul 19, 2022 at 02:43:59PM +0530, Bharath Rupireddy wrote:
> > > Done. PSA v5 patch set.
> >
> > LGTM. I've marked this as ready-for-committer.
>
> I find the following sentense in config.sgml. "Specifies the minimum
> size of past log file segments kept in the pg_wal directory"
>
> postgresql.conf.sample contains "logfile segment" in a few lines.
Done. PSA v6 patch set.
Regards,
Bharath Rupireddy.
Attachments:
[application/octet-stream] v6-0001-Use-WAL-segment-instead-of-log-segment.patch (9.0K, ../../CALj2ACV0ux_AeCV-xJkA6rbmsdN8KTHGOCx8vJCAa=DEtmgqaA@mail.gmail.com/2-v6-0001-Use-WAL-segment-instead-of-log-segment.patch)
download | inline diff:
From 6094c80f28c64904ede361672521be6b86993adb Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 20 Jul 2022 04:25:10 +0000
Subject: [PATCH v6] Use "WAL segment" instead of "log segment"
We are using "log segment" in various user-facing messages, the
term "log" can mean server logs as well. The "WAL segment" suits
well here and it is consistently used across the other user-facing
messages.
Author: Bharath Rupireddy
---
src/backend/access/transam/xlogreader.c | 10 +++++-----
src/backend/access/transam/xlogrecovery.c | 6 +++---
src/backend/access/transam/xlogutils.c | 4 ++--
src/backend/replication/walreceiver.c | 6 +++---
src/bin/pg_resetwal/pg_resetwal.c | 2 +-
src/bin/pg_upgrade/controldata.c | 2 +-
src/bin/pg_waldump/pg_waldump.c | 4 ++--
7 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f3dc4b7797..58f1a32b00 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1209,7 +1209,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid magic number %04X in log segment %s, offset %u",
+ "invalid magic number %04X in WAL segment %s, offset %u",
hdr->xlp_magic,
fname,
offset);
@@ -1223,7 +1223,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1264,7 +1264,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
/* hmm, first page of file doesn't have a long header? */
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1283,7 +1283,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "unexpected pageaddr %X/%X in log segment %s, offset %u",
+ "unexpected pageaddr %X/%X in WAL segment %s, offset %u",
LSN_FORMAT_ARGS(hdr->xlp_pageaddr),
fname,
offset);
@@ -1308,7 +1308,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u",
+ "out-of-sequence timeline ID %u (after %u) in WAL segment %s, offset %u",
hdr->xlp_tli,
state->latestPageTLI,
fname,
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..306a9f40e9 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -3018,7 +3018,7 @@ ReadRecord(XLogPrefetcher *xlogprefetcher, int emode,
XLogFileName(fname, xlogreader->seg.ws_tli, segno,
wal_segment_size);
ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
- (errmsg("unexpected timeline ID %u in log segment %s, offset %u",
+ (errmsg("unexpected timeline ID %u in WAL segment %s, offset %u",
xlogreader->latestPageTLI,
fname,
offset)));
@@ -3223,13 +3223,13 @@ retry:
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %u: %m",
+ errmsg("could not read from WAL segment %s, offset %u: %m",
fname, readOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %u: read %d of %zu",
+ errmsg("could not read from WAL segment %s, offset %u: read %d of %zu",
fname, readOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 0cda22597f..9e3a000768 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1049,14 +1049,14 @@ WALReadRaiseError(WALReadError *errinfo)
errno = errinfo->wre_errno;
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %d: %m",
+ errmsg("could not read from WAL segment %s, offset %d: %m",
fname, errinfo->wre_off)));
}
else if (errinfo->wre_read == 0)
{
ereport(ERROR,
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %d: read %d of %d",
+ errmsg("could not read from WAL segment %s, offset %d: read %d of %d",
fname, errinfo->wre_off, errinfo->wre_read,
errinfo->wre_req)));
}
diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c
index 3d37c1fe62..3767466ef3 100644
--- a/src/backend/replication/walreceiver.c
+++ b/src/backend/replication/walreceiver.c
@@ -616,7 +616,7 @@ WalReceiverMain(void)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
@@ -930,7 +930,7 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr, TimeLineID tli)
errno = save_errno;
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not write to log segment %s "
+ errmsg("could not write to WAL segment %s "
"at offset %u, length %lu: %m",
xlogfname, startoff, (unsigned long) segbytes)));
}
@@ -1042,7 +1042,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index d4772a2965..7adf79eeed 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -788,7 +788,7 @@ PrintNewControlValues(void)
XLogFileName(fname, ControlFile.checkPointCopy.ThisTimeLineID,
newXlogSegNo, WalSegSz);
- printf(_("First log segment after reset: %s\n"), fname);
+ printf(_("First WAL segment after reset: %s\n"), fname);
if (set_mxid != 0)
{
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 07de918358..678e8ebf6b 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -350,7 +350,7 @@ get_control_data(ClusterInfo *cluster, bool live_check)
cluster->controldata.chkpnt_nxtmxoff = str2uint(p);
got_mxoff = true;
}
- else if ((p = strstr(bufin, "First log segment after reset:")) != NULL)
+ else if ((p = strstr(bufin, "First WAL segment after reset:")) != NULL)
{
/* Skip the colon and any whitespace after it */
p = strchr(p, ':');
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6528113628..4eebeadc8c 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -667,7 +667,7 @@ usage(void)
printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n"
" valid names are main, fsm, vm, init\n"));
printf(_(" -n, --limit=N number of records to display\n"));
- printf(_(" -p, --path=PATH directory in which to find log segment files or a\n"
+ printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n"
" directory with a ./pg_wal that contains such files\n"
" (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n"));
printf(_(" -q, --quiet do not print any output, except for errors\n"));
@@ -675,7 +675,7 @@ usage(void)
" use --rmgr=list to list valid resource manager names\n"));
printf(_(" -R, --relation=T/D/R only show records that modify blocks in relation T/D/R\n"));
printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n"));
- printf(_(" -t, --timeline=TLI timeline from which to read log records\n"
+ printf(_(" -t, --timeline=TLI timeline from which to read WAL records\n"
" (default: 1 or the value used in STARTSEG)\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -w, --fullpage only show records with a full page write\n"));
--
2.25.1
[application/octet-stream] v6-0002-Consistently-use-WAL-file-s-in-docs.patch (18.6K, ../../CALj2ACV0ux_AeCV-xJkA6rbmsdN8KTHGOCx8vJCAa=DEtmgqaA@mail.gmail.com/3-v6-0002-Consistently-use-WAL-file-s-in-docs.patch)
download | inline diff:
From 2a89079004c0f85040bd3d504874bff1e9e34f39 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 20 Jul 2022 04:29:46 +0000
Subject: [PATCH v6] Consistently use "WAL file(s)" in docs
Authors: Kyotaro Horiguchi, Bharath Rupireddy
---
doc/src/sgml/backup.sgml | 14 ++++----
doc/src/sgml/config.sgml | 4 +--
doc/src/sgml/ref/pg_waldump.sgml | 10 +++---
doc/src/sgml/wal.sgml | 60 ++++++++++++++++----------------
4 files changed, 44 insertions(+), 44 deletions(-)
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 73a774d3d7..cc5ae59ac2 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1095,7 +1095,7 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
require that you have enough free space on your system to hold two
copies of your existing database. If you do not have enough space,
you should at least save the contents of the cluster's <filename>pg_wal</filename>
- subdirectory, as it might contain logs which
+ subdirectory, as it might contain WAL files which
were not archived before the system went down.
</para>
</listitem>
@@ -1173,8 +1173,8 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
which tells <productname>PostgreSQL</productname> how to retrieve archived
WAL file segments. Like the <varname>archive_command</varname>, this is
a shell command string. It can contain <literal>%f</literal>, which is
- replaced by the name of the desired log file, and <literal>%p</literal>,
- which is replaced by the path name to copy the log file to.
+ replaced by the name of the desired WAL file, and <literal>%p</literal>,
+ which is replaced by the path name to copy the WAL file to.
(The path name is relative to the current working directory,
i.e., the cluster's data directory.)
Write <literal>%%</literal> if you need to embed an actual <literal>%</literal>
@@ -1462,9 +1462,9 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
<link linkend="sql-createtablespace"><command>CREATE TABLESPACE</command></link>
commands are WAL-logged with the literal absolute path, and will
therefore be replayed as tablespace creations with the same
- absolute path. This might be undesirable if the log is being
+ absolute path. This might be undesirable if the WAL is being
replayed on a different machine. It can be dangerous even if the
- log is being replayed on the same machine, but into a new data
+ WAL is being replayed on the same machine, but into a new data
directory: the replay will still overwrite the contents of the
original tablespace. To avoid potential gotchas of this sort,
the best practice is to take a new base backup after creating or
@@ -1481,11 +1481,11 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
we might need to fix partially-written disk pages. Depending on
your system hardware and software, the risk of partial writes might
be small enough to ignore, in which case you can significantly
- reduce the total volume of archived logs by turning off page
+ reduce the total volume of archived WAL files by turning off page
snapshots using the <xref linkend="guc-full-page-writes"/>
parameter. (Read the notes and warnings in <xref linkend="wal"/>
before you do so.) Turning off page snapshots does not prevent
- use of the logs for PITR operations. An area for future
+ use of the WAL for PITR operations. An area for future
development is to compress archived WAL data by removing
unnecessary page copies even when <varname>full_page_writes</varname> is
on. In the meantime, administrators might wish to reduce the number
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 37fd80388c..8275e557ef 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4228,7 +4228,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</term>
<listitem>
<para>
- Specifies the minimum size of past log file segments kept in the
+ Specifies the minimum size of past WAL files kept in the
<filename>pg_wal</filename>
directory, in case a standby server needs to fetch them for streaming
replication. If a standby
@@ -4821,7 +4821,7 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
needs to control the amount of time to wait for new WAL data to be
available. For example, in archive recovery, it is possible to
make the recovery more responsive in the detection of a new WAL
- log file by reducing the value of this parameter. On a system with
+ file by reducing the value of this parameter. On a system with
low WAL activity, increasing it reduces the amount of requests necessary
to access WAL archives, something useful for example in cloud
environments where the number of times an infrastructure is accessed
diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml
index 57746d9421..2e2166bb6f 100644
--- a/doc/src/sgml/ref/pg_waldump.sgml
+++ b/doc/src/sgml/ref/pg_waldump.sgml
@@ -53,7 +53,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">startseg</replaceable></term>
<listitem>
<para>
- Start reading at the specified log segment file. This implicitly determines
+ Start reading at the specified WAL segment file. This implicitly determines
the path in which files will be searched for, and the timeline to use.
</para>
</listitem>
@@ -63,7 +63,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">endseg</replaceable></term>
<listitem>
<para>
- Stop after reading the specified log segment file.
+ Stop after reading the specified WAL segment file.
</para>
</listitem>
</varlistentry>
@@ -141,7 +141,7 @@ PostgreSQL documentation
<term><option>--path=<replaceable>path</replaceable></option></term>
<listitem>
<para>
- Specifies a directory to search for log segment files or a
+ Specifies a directory to search for WAL segment files or a
directory with a <literal>pg_wal</literal> subdirectory that
contains such files. The default is to search in the current
directory, the <literal>pg_wal</literal> subdirectory of the
@@ -203,7 +203,7 @@ PostgreSQL documentation
<listitem>
<para>
WAL location at which to start reading. The default is to start reading
- the first valid log record found in the earliest file found.
+ the first valid WAL record found in the earliest file found.
</para>
</listitem>
</varlistentry>
@@ -213,7 +213,7 @@ PostgreSQL documentation
<term><option>--timeline=<replaceable>timeline</replaceable></option></term>
<listitem>
<para>
- Timeline from which to read log records. The default is to use the
+ Timeline from which to read WAL records. The default is to use the
value in <replaceable>startseg</replaceable>, if that is specified; otherwise, the
default is 1.
</para>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 01f7379ebb..30842c0396 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -297,12 +297,12 @@
transaction processing. Briefly, <acronym>WAL</acronym>'s central
concept is that changes to data files (where tables and indexes
reside) must be written only after those changes have been logged,
- that is, after log records describing the changes have been flushed
+ that is, after WAL records describing the changes have been flushed
to permanent storage. If we follow this procedure, we do not need
to flush data pages to disk on every transaction commit, because we
know that in the event of a crash we will be able to recover the
database using the log: any changes that have not been applied to
- the data pages can be redone from the log records. (This is
+ the data pages can be redone from the WAL records. (This is
roll-forward recovery, also known as REDO.)
</para>
@@ -323,15 +323,15 @@
<para>
Using <acronym>WAL</acronym> results in a
- significantly reduced number of disk writes, because only the log
+ significantly reduced number of disk writes, because only the WAL
file needs to be flushed to disk to guarantee that a transaction is
committed, rather than every data file changed by the transaction.
- The log file is written sequentially,
- and so the cost of syncing the log is much less than the cost of
+ The WAL file is written sequentially,
+ and so the cost of syncing the WAL is much less than the cost of
flushing the data pages. This is especially true for servers
handling many small transactions touching different parts of the data
store. Furthermore, when the server is processing many small concurrent
- transactions, one <function>fsync</function> of the log file may
+ transactions, one <function>fsync</function> of the WAL file may
suffice to commit many transactions.
</para>
@@ -341,10 +341,10 @@
linkend="continuous-archiving"/>. By archiving the WAL data we can support
reverting to any time instant covered by the available WAL data:
we simply install a prior physical backup of the database, and
- replay the WAL log just as far as the desired time. What's more,
+ replay the WAL just as far as the desired time. What's more,
the physical backup doesn't have to be an instantaneous snapshot
of the database state — if it is made over some period of time,
- then replaying the WAL log for that period will fix any internal
+ then replaying the WAL for that period will fix any internal
inconsistencies.
</para>
</sect1>
@@ -497,15 +497,15 @@
that the heap and index data files have been updated with all
information written before that checkpoint. At checkpoint time, all
dirty data pages are flushed to disk and a special checkpoint record is
- written to the log file. (The change records were previously flushed
+ written to the WAL file. (The change records were previously flushed
to the <acronym>WAL</acronym> files.)
In the event of a crash, the crash recovery procedure looks at the latest
- checkpoint record to determine the point in the log (known as the redo
+ checkpoint record to determine the point in the WAL (known as the redo
record) from which it should start the REDO operation. Any changes made to
data files before that point are guaranteed to be already on disk.
- Hence, after a checkpoint, log segments preceding the one containing
+ Hence, after a checkpoint, WAL segments preceding the one containing
the redo record are no longer needed and can be recycled or removed. (When
- <acronym>WAL</acronym> archiving is being done, the log segments must be
+ <acronym>WAL</acronym> archiving is being done, the WAL segments must be
archived before being recycled or removed.)
</para>
@@ -544,7 +544,7 @@
another factor to consider. To ensure data page consistency,
the first modification of a data page after each checkpoint results in
logging the entire page content. In that case,
- a smaller checkpoint interval increases the volume of output to the WAL log,
+ a smaller checkpoint interval increases the volume of output to the WAL,
partially negating the goal of using a smaller interval,
and in any case causing more disk I/O.
</para>
@@ -614,10 +614,10 @@
<para>
The number of WAL segment files in <filename>pg_wal</filename> directory depends on
<varname>min_wal_size</varname>, <varname>max_wal_size</varname> and
- the amount of WAL generated in previous checkpoint cycles. When old log
+ the amount of WAL generated in previous checkpoint cycles. When old WAL
segment files are no longer needed, they are removed or recycled (that is,
renamed to become future segments in the numbered sequence). If, due to a
- short-term peak of log output rate, <varname>max_wal_size</varname> is
+ short-term peak of WAL output rate, <varname>max_wal_size</varname> is
exceeded, the unneeded segment files will be removed until the system
gets back under this limit. Below that limit, the system recycles enough
WAL files to cover the estimated need until the next checkpoint, and
@@ -650,7 +650,7 @@
which are similar to checkpoints in normal operation: the server forces
all its state to disk, updates the <filename>pg_control</filename> file to
indicate that the already-processed WAL data need not be scanned again,
- and then recycles any old log segment files in the <filename>pg_wal</filename>
+ and then recycles any old WAL segment files in the <filename>pg_wal</filename>
directory.
Restartpoints can't be performed more frequently than checkpoints on the
primary because restartpoints can only be performed at checkpoint records.
@@ -676,12 +676,12 @@
insertion) at a time when an exclusive lock is held on affected
data pages, so the operation needs to be as fast as possible. What
is worse, writing <acronym>WAL</acronym> buffers might also force the
- creation of a new log segment, which takes even more
+ creation of a new WAL segment, which takes even more
time. Normally, <acronym>WAL</acronym> buffers should be written
and flushed by an <function>XLogFlush</function> request, which is
made, for the most part, at transaction commit time to ensure that
transaction records are flushed to permanent storage. On systems
- with high log output, <function>XLogFlush</function> requests might
+ with high WAL output, <function>XLogFlush</function> requests might
not occur often enough to prevent <function>XLogInsertRecord</function>
from having to do writes. On such systems
one should increase the number of <acronym>WAL</acronym> buffers by
@@ -724,7 +724,7 @@
<varname>commit_delay</varname>, so this value is recommended as the
starting point to use when optimizing for a particular workload. While
tuning <varname>commit_delay</varname> is particularly useful when the
- WAL log is stored on high-latency rotating disks, benefits can be
+ WAL is stored on high-latency rotating disks, benefits can be
significant even on storage media with very fast sync times, such as
solid-state drives or RAID arrays with a battery-backed write cache;
but this should definitely be tested against a representative workload.
@@ -828,16 +828,16 @@
<para>
<acronym>WAL</acronym> is automatically enabled; no action is
required from the administrator except ensuring that the
- disk-space requirements for the <acronym>WAL</acronym> logs are met,
+ disk-space requirements for the <acronym>WAL</acronym> files are met,
and that any necessary tuning is done (see <xref
linkend="wal-configuration"/>).
</para>
<para>
<acronym>WAL</acronym> records are appended to the <acronym>WAL</acronym>
- logs as each new record is written. The insert position is described by
+ files as each new record is written. The insert position is described by
a Log Sequence Number (<acronym>LSN</acronym>) that is a byte offset into
- the logs, increasing monotonically with each new record.
+ the WAL, increasing monotonically with each new record.
<acronym>LSN</acronym> values are returned as the datatype
<link linkend="datatype-pg-lsn"><type>pg_lsn</type></link>. Values can be
compared to calculate the volume of <acronym>WAL</acronym> data that
@@ -846,12 +846,12 @@
</para>
<para>
- <acronym>WAL</acronym> logs are stored in the directory
+ <acronym>WAL</acronym> files are stored in the directory
<filename>pg_wal</filename> under the data directory, as a set of
segment files, normally each 16 MB in size (but the size can be changed
by altering the <option>--wal-segsize</option> <application>initdb</application> option). Each segment is
divided into pages, normally 8 kB each (this size can be changed via the
- <option>--with-wal-blocksize</option> configure option). The log record headers
+ <option>--with-wal-blocksize</option> configure option). The WAL record headers
are described in <filename>access/xlogrecord.h</filename>; the record
content is dependent on the type of event that is being logged. Segment
files are given ever-increasing numbers as names, starting at
@@ -861,7 +861,7 @@
</para>
<para>
- It is advantageous if the log is located on a different disk from the
+ It is advantageous if the WAL is located on a different disk from the
main database files. This can be achieved by moving the
<filename>pg_wal</filename> directory to another location (while the server
is shut down, of course) and creating a symbolic link from the
@@ -877,19 +877,19 @@
on the disk. A power failure in such a situation might lead to
irrecoverable data corruption. Administrators should try to ensure
that disks holding <productname>PostgreSQL</productname>'s
- <acronym>WAL</acronym> log files do not make such false reports.
+ <acronym>WAL</acronym> files do not make such false reports.
(See <xref linkend="wal-reliability"/>.)
</para>
<para>
- After a checkpoint has been made and the log flushed, the
+ After a checkpoint has been made and the WAL flushed, the
checkpoint's position is saved in the file
<filename>pg_control</filename>. Therefore, at the start of recovery,
the server first reads <filename>pg_control</filename> and
then the checkpoint record; then it performs the REDO operation by
- scanning forward from the log location indicated in the checkpoint
+ scanning forward from the WAL location indicated in the checkpoint
record. Because the entire content of data pages is saved in the
- log on the first page modification after a checkpoint (assuming
+ WAL on the first page modification after a checkpoint (assuming
<xref linkend="guc-full-page-writes"/> is not disabled), all pages
changed since the checkpoint will be restored to a consistent
state.
@@ -897,7 +897,7 @@
<para>
To deal with the case where <filename>pg_control</filename> is
- corrupt, we should support the possibility of scanning existing log
+ corrupt, we should support the possibility of scanning existing WAL
segments in reverse order — newest to oldest — in order to find the
latest checkpoint. This has not been implemented yet.
<filename>pg_control</filename> is small enough (less than one disk page)
--
2.25.1
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-20 01:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 04:32 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
@ 2022-07-20 07:25 ` Kyotaro Horiguchi <[email protected]>
2022-07-20 11:55 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Kyotaro Horiguchi @ 2022-07-20 07:25 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]
At Wed, 20 Jul 2022 10:02:22 +0530, Bharath Rupireddy <[email protected]> wrote in
> Done. PSA v6 patch set.
Thanks!
- Specifies the minimum size of past log file segments kept in the
+ Specifies the minimum size of past WAL files kept in the
- log file by reducing the value of this parameter. On a system with
+ file by reducing the value of this parameter. On a system with
Looks fine. And postgresq.conf.sample has the following lines:
#archive_library = '' # library to use to archive a logfile segment
#archive_command = '' # command to use to archive a logfile segment
#archive_timeout = 0 # force a logfile segment switch after this
#restore_command = '' # command to use to restore an archived logfile segment
Aren't they need the same fix?
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-20 01:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 04:32 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-20 07:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
@ 2022-07-20 11:55 ` Bharath Rupireddy <[email protected]>
2022-07-21 04:20 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Bharath Rupireddy @ 2022-07-20 11:55 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: Nathan Bossart <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jul 20, 2022 at 12:55 PM Kyotaro Horiguchi
<[email protected]> wrote:
>
> At Wed, 20 Jul 2022 10:02:22 +0530, Bharath Rupireddy <[email protected]> wrote in
> > Done. PSA v6 patch set.
>
> Thanks!
>
> - Specifies the minimum size of past log file segments kept in the
> + Specifies the minimum size of past WAL files kept in the
>
> - log file by reducing the value of this parameter. On a system with
> + file by reducing the value of this parameter. On a system with
>
> Looks fine. And postgresq.conf.sample has the following lines:
>
> #archive_library = '' # library to use to archive a logfile segment
>
> #archive_command = '' # command to use to archive a logfile segment
>
> #archive_timeout = 0 # force a logfile segment switch after this
>
> #restore_command = '' # command to use to restore an archived logfile segment
>
> Aren't they need the same fix?
Indeed. Thanks. Now, they are in sync with their peers in .conf.sample
file as well as description in guc.c.
PSA v7 patch set.
Regards,
Bharath Rupireddy.
Attachments:
[application/octet-stream] v7-0001-Use-WAL-segment-instead-of-log-segment.patch (9.0K, ../../CALj2ACW8YwHiCr5PqvW8625Wx1ZaaVumHY-uzS1JDabvbrgVCw@mail.gmail.com/2-v7-0001-Use-WAL-segment-instead-of-log-segment.patch)
download | inline diff:
From 6094c80f28c64904ede361672521be6b86993adb Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 20 Jul 2022 04:25:10 +0000
Subject: [PATCH v7] Use "WAL segment" instead of "log segment"
We are using "log segment" in various user-facing messages, the
term "log" can mean server logs as well. The "WAL segment" suits
well here and it is consistently used across the other user-facing
messages.
Author: Bharath Rupireddy
---
src/backend/access/transam/xlogreader.c | 10 +++++-----
src/backend/access/transam/xlogrecovery.c | 6 +++---
src/backend/access/transam/xlogutils.c | 4 ++--
src/backend/replication/walreceiver.c | 6 +++---
src/bin/pg_resetwal/pg_resetwal.c | 2 +-
src/bin/pg_upgrade/controldata.c | 2 +-
src/bin/pg_waldump/pg_waldump.c | 4 ++--
7 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f3dc4b7797..58f1a32b00 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1209,7 +1209,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid magic number %04X in log segment %s, offset %u",
+ "invalid magic number %04X in WAL segment %s, offset %u",
hdr->xlp_magic,
fname,
offset);
@@ -1223,7 +1223,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1264,7 +1264,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
/* hmm, first page of file doesn't have a long header? */
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1283,7 +1283,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "unexpected pageaddr %X/%X in log segment %s, offset %u",
+ "unexpected pageaddr %X/%X in WAL segment %s, offset %u",
LSN_FORMAT_ARGS(hdr->xlp_pageaddr),
fname,
offset);
@@ -1308,7 +1308,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u",
+ "out-of-sequence timeline ID %u (after %u) in WAL segment %s, offset %u",
hdr->xlp_tli,
state->latestPageTLI,
fname,
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..306a9f40e9 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -3018,7 +3018,7 @@ ReadRecord(XLogPrefetcher *xlogprefetcher, int emode,
XLogFileName(fname, xlogreader->seg.ws_tli, segno,
wal_segment_size);
ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
- (errmsg("unexpected timeline ID %u in log segment %s, offset %u",
+ (errmsg("unexpected timeline ID %u in WAL segment %s, offset %u",
xlogreader->latestPageTLI,
fname,
offset)));
@@ -3223,13 +3223,13 @@ retry:
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %u: %m",
+ errmsg("could not read from WAL segment %s, offset %u: %m",
fname, readOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %u: read %d of %zu",
+ errmsg("could not read from WAL segment %s, offset %u: read %d of %zu",
fname, readOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 0cda22597f..9e3a000768 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1049,14 +1049,14 @@ WALReadRaiseError(WALReadError *errinfo)
errno = errinfo->wre_errno;
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %d: %m",
+ errmsg("could not read from WAL segment %s, offset %d: %m",
fname, errinfo->wre_off)));
}
else if (errinfo->wre_read == 0)
{
ereport(ERROR,
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %d: read %d of %d",
+ errmsg("could not read from WAL segment %s, offset %d: read %d of %d",
fname, errinfo->wre_off, errinfo->wre_read,
errinfo->wre_req)));
}
diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c
index 3d37c1fe62..3767466ef3 100644
--- a/src/backend/replication/walreceiver.c
+++ b/src/backend/replication/walreceiver.c
@@ -616,7 +616,7 @@ WalReceiverMain(void)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
@@ -930,7 +930,7 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr, TimeLineID tli)
errno = save_errno;
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not write to log segment %s "
+ errmsg("could not write to WAL segment %s "
"at offset %u, length %lu: %m",
xlogfname, startoff, (unsigned long) segbytes)));
}
@@ -1042,7 +1042,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index d4772a2965..7adf79eeed 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -788,7 +788,7 @@ PrintNewControlValues(void)
XLogFileName(fname, ControlFile.checkPointCopy.ThisTimeLineID,
newXlogSegNo, WalSegSz);
- printf(_("First log segment after reset: %s\n"), fname);
+ printf(_("First WAL segment after reset: %s\n"), fname);
if (set_mxid != 0)
{
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 07de918358..678e8ebf6b 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -350,7 +350,7 @@ get_control_data(ClusterInfo *cluster, bool live_check)
cluster->controldata.chkpnt_nxtmxoff = str2uint(p);
got_mxoff = true;
}
- else if ((p = strstr(bufin, "First log segment after reset:")) != NULL)
+ else if ((p = strstr(bufin, "First WAL segment after reset:")) != NULL)
{
/* Skip the colon and any whitespace after it */
p = strchr(p, ':');
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6528113628..4eebeadc8c 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -667,7 +667,7 @@ usage(void)
printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n"
" valid names are main, fsm, vm, init\n"));
printf(_(" -n, --limit=N number of records to display\n"));
- printf(_(" -p, --path=PATH directory in which to find log segment files or a\n"
+ printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n"
" directory with a ./pg_wal that contains such files\n"
" (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n"));
printf(_(" -q, --quiet do not print any output, except for errors\n"));
@@ -675,7 +675,7 @@ usage(void)
" use --rmgr=list to list valid resource manager names\n"));
printf(_(" -R, --relation=T/D/R only show records that modify blocks in relation T/D/R\n"));
printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n"));
- printf(_(" -t, --timeline=TLI timeline from which to read log records\n"
+ printf(_(" -t, --timeline=TLI timeline from which to read WAL records\n"
" (default: 1 or the value used in STARTSEG)\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -w, --fullpage only show records with a full page write\n"));
--
2.25.1
[application/octet-stream] v7-0002-Consistently-use-WAL-file-s-in-docs.patch (20.0K, ../../CALj2ACW8YwHiCr5PqvW8625Wx1ZaaVumHY-uzS1JDabvbrgVCw@mail.gmail.com/3-v7-0002-Consistently-use-WAL-file-s-in-docs.patch)
download | inline diff:
From 0e662610e1335a16529d9da6bb45cd9d410faa45 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Wed, 20 Jul 2022 11:53:03 +0000
Subject: [PATCH v7] Consistently use "WAL file(s)" in docs
Authors: Kyotaro Horiguchi, Bharath Rupireddy
---
doc/src/sgml/backup.sgml | 14 ++---
doc/src/sgml/config.sgml | 4 +-
doc/src/sgml/ref/pg_waldump.sgml | 10 ++--
doc/src/sgml/wal.sgml | 60 +++++++++----------
src/backend/utils/misc/postgresql.conf.sample | 8 +--
5 files changed, 48 insertions(+), 48 deletions(-)
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 73a774d3d7..cc5ae59ac2 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1095,7 +1095,7 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
require that you have enough free space on your system to hold two
copies of your existing database. If you do not have enough space,
you should at least save the contents of the cluster's <filename>pg_wal</filename>
- subdirectory, as it might contain logs which
+ subdirectory, as it might contain WAL files which
were not archived before the system went down.
</para>
</listitem>
@@ -1173,8 +1173,8 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
which tells <productname>PostgreSQL</productname> how to retrieve archived
WAL file segments. Like the <varname>archive_command</varname>, this is
a shell command string. It can contain <literal>%f</literal>, which is
- replaced by the name of the desired log file, and <literal>%p</literal>,
- which is replaced by the path name to copy the log file to.
+ replaced by the name of the desired WAL file, and <literal>%p</literal>,
+ which is replaced by the path name to copy the WAL file to.
(The path name is relative to the current working directory,
i.e., the cluster's data directory.)
Write <literal>%%</literal> if you need to embed an actual <literal>%</literal>
@@ -1462,9 +1462,9 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
<link linkend="sql-createtablespace"><command>CREATE TABLESPACE</command></link>
commands are WAL-logged with the literal absolute path, and will
therefore be replayed as tablespace creations with the same
- absolute path. This might be undesirable if the log is being
+ absolute path. This might be undesirable if the WAL is being
replayed on a different machine. It can be dangerous even if the
- log is being replayed on the same machine, but into a new data
+ WAL is being replayed on the same machine, but into a new data
directory: the replay will still overwrite the contents of the
original tablespace. To avoid potential gotchas of this sort,
the best practice is to take a new base backup after creating or
@@ -1481,11 +1481,11 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
we might need to fix partially-written disk pages. Depending on
your system hardware and software, the risk of partial writes might
be small enough to ignore, in which case you can significantly
- reduce the total volume of archived logs by turning off page
+ reduce the total volume of archived WAL files by turning off page
snapshots using the <xref linkend="guc-full-page-writes"/>
parameter. (Read the notes and warnings in <xref linkend="wal"/>
before you do so.) Turning off page snapshots does not prevent
- use of the logs for PITR operations. An area for future
+ use of the WAL for PITR operations. An area for future
development is to compress archived WAL data by removing
unnecessary page copies even when <varname>full_page_writes</varname> is
on. In the meantime, administrators might wish to reduce the number
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 37fd80388c..8275e557ef 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4228,7 +4228,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</term>
<listitem>
<para>
- Specifies the minimum size of past log file segments kept in the
+ Specifies the minimum size of past WAL files kept in the
<filename>pg_wal</filename>
directory, in case a standby server needs to fetch them for streaming
replication. If a standby
@@ -4821,7 +4821,7 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
needs to control the amount of time to wait for new WAL data to be
available. For example, in archive recovery, it is possible to
make the recovery more responsive in the detection of a new WAL
- log file by reducing the value of this parameter. On a system with
+ file by reducing the value of this parameter. On a system with
low WAL activity, increasing it reduces the amount of requests necessary
to access WAL archives, something useful for example in cloud
environments where the number of times an infrastructure is accessed
diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml
index 57746d9421..2e2166bb6f 100644
--- a/doc/src/sgml/ref/pg_waldump.sgml
+++ b/doc/src/sgml/ref/pg_waldump.sgml
@@ -53,7 +53,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">startseg</replaceable></term>
<listitem>
<para>
- Start reading at the specified log segment file. This implicitly determines
+ Start reading at the specified WAL segment file. This implicitly determines
the path in which files will be searched for, and the timeline to use.
</para>
</listitem>
@@ -63,7 +63,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">endseg</replaceable></term>
<listitem>
<para>
- Stop after reading the specified log segment file.
+ Stop after reading the specified WAL segment file.
</para>
</listitem>
</varlistentry>
@@ -141,7 +141,7 @@ PostgreSQL documentation
<term><option>--path=<replaceable>path</replaceable></option></term>
<listitem>
<para>
- Specifies a directory to search for log segment files or a
+ Specifies a directory to search for WAL segment files or a
directory with a <literal>pg_wal</literal> subdirectory that
contains such files. The default is to search in the current
directory, the <literal>pg_wal</literal> subdirectory of the
@@ -203,7 +203,7 @@ PostgreSQL documentation
<listitem>
<para>
WAL location at which to start reading. The default is to start reading
- the first valid log record found in the earliest file found.
+ the first valid WAL record found in the earliest file found.
</para>
</listitem>
</varlistentry>
@@ -213,7 +213,7 @@ PostgreSQL documentation
<term><option>--timeline=<replaceable>timeline</replaceable></option></term>
<listitem>
<para>
- Timeline from which to read log records. The default is to use the
+ Timeline from which to read WAL records. The default is to use the
value in <replaceable>startseg</replaceable>, if that is specified; otherwise, the
default is 1.
</para>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 01f7379ebb..30842c0396 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -297,12 +297,12 @@
transaction processing. Briefly, <acronym>WAL</acronym>'s central
concept is that changes to data files (where tables and indexes
reside) must be written only after those changes have been logged,
- that is, after log records describing the changes have been flushed
+ that is, after WAL records describing the changes have been flushed
to permanent storage. If we follow this procedure, we do not need
to flush data pages to disk on every transaction commit, because we
know that in the event of a crash we will be able to recover the
database using the log: any changes that have not been applied to
- the data pages can be redone from the log records. (This is
+ the data pages can be redone from the WAL records. (This is
roll-forward recovery, also known as REDO.)
</para>
@@ -323,15 +323,15 @@
<para>
Using <acronym>WAL</acronym> results in a
- significantly reduced number of disk writes, because only the log
+ significantly reduced number of disk writes, because only the WAL
file needs to be flushed to disk to guarantee that a transaction is
committed, rather than every data file changed by the transaction.
- The log file is written sequentially,
- and so the cost of syncing the log is much less than the cost of
+ The WAL file is written sequentially,
+ and so the cost of syncing the WAL is much less than the cost of
flushing the data pages. This is especially true for servers
handling many small transactions touching different parts of the data
store. Furthermore, when the server is processing many small concurrent
- transactions, one <function>fsync</function> of the log file may
+ transactions, one <function>fsync</function> of the WAL file may
suffice to commit many transactions.
</para>
@@ -341,10 +341,10 @@
linkend="continuous-archiving"/>. By archiving the WAL data we can support
reverting to any time instant covered by the available WAL data:
we simply install a prior physical backup of the database, and
- replay the WAL log just as far as the desired time. What's more,
+ replay the WAL just as far as the desired time. What's more,
the physical backup doesn't have to be an instantaneous snapshot
of the database state — if it is made over some period of time,
- then replaying the WAL log for that period will fix any internal
+ then replaying the WAL for that period will fix any internal
inconsistencies.
</para>
</sect1>
@@ -497,15 +497,15 @@
that the heap and index data files have been updated with all
information written before that checkpoint. At checkpoint time, all
dirty data pages are flushed to disk and a special checkpoint record is
- written to the log file. (The change records were previously flushed
+ written to the WAL file. (The change records were previously flushed
to the <acronym>WAL</acronym> files.)
In the event of a crash, the crash recovery procedure looks at the latest
- checkpoint record to determine the point in the log (known as the redo
+ checkpoint record to determine the point in the WAL (known as the redo
record) from which it should start the REDO operation. Any changes made to
data files before that point are guaranteed to be already on disk.
- Hence, after a checkpoint, log segments preceding the one containing
+ Hence, after a checkpoint, WAL segments preceding the one containing
the redo record are no longer needed and can be recycled or removed. (When
- <acronym>WAL</acronym> archiving is being done, the log segments must be
+ <acronym>WAL</acronym> archiving is being done, the WAL segments must be
archived before being recycled or removed.)
</para>
@@ -544,7 +544,7 @@
another factor to consider. To ensure data page consistency,
the first modification of a data page after each checkpoint results in
logging the entire page content. In that case,
- a smaller checkpoint interval increases the volume of output to the WAL log,
+ a smaller checkpoint interval increases the volume of output to the WAL,
partially negating the goal of using a smaller interval,
and in any case causing more disk I/O.
</para>
@@ -614,10 +614,10 @@
<para>
The number of WAL segment files in <filename>pg_wal</filename> directory depends on
<varname>min_wal_size</varname>, <varname>max_wal_size</varname> and
- the amount of WAL generated in previous checkpoint cycles. When old log
+ the amount of WAL generated in previous checkpoint cycles. When old WAL
segment files are no longer needed, they are removed or recycled (that is,
renamed to become future segments in the numbered sequence). If, due to a
- short-term peak of log output rate, <varname>max_wal_size</varname> is
+ short-term peak of WAL output rate, <varname>max_wal_size</varname> is
exceeded, the unneeded segment files will be removed until the system
gets back under this limit. Below that limit, the system recycles enough
WAL files to cover the estimated need until the next checkpoint, and
@@ -650,7 +650,7 @@
which are similar to checkpoints in normal operation: the server forces
all its state to disk, updates the <filename>pg_control</filename> file to
indicate that the already-processed WAL data need not be scanned again,
- and then recycles any old log segment files in the <filename>pg_wal</filename>
+ and then recycles any old WAL segment files in the <filename>pg_wal</filename>
directory.
Restartpoints can't be performed more frequently than checkpoints on the
primary because restartpoints can only be performed at checkpoint records.
@@ -676,12 +676,12 @@
insertion) at a time when an exclusive lock is held on affected
data pages, so the operation needs to be as fast as possible. What
is worse, writing <acronym>WAL</acronym> buffers might also force the
- creation of a new log segment, which takes even more
+ creation of a new WAL segment, which takes even more
time. Normally, <acronym>WAL</acronym> buffers should be written
and flushed by an <function>XLogFlush</function> request, which is
made, for the most part, at transaction commit time to ensure that
transaction records are flushed to permanent storage. On systems
- with high log output, <function>XLogFlush</function> requests might
+ with high WAL output, <function>XLogFlush</function> requests might
not occur often enough to prevent <function>XLogInsertRecord</function>
from having to do writes. On such systems
one should increase the number of <acronym>WAL</acronym> buffers by
@@ -724,7 +724,7 @@
<varname>commit_delay</varname>, so this value is recommended as the
starting point to use when optimizing for a particular workload. While
tuning <varname>commit_delay</varname> is particularly useful when the
- WAL log is stored on high-latency rotating disks, benefits can be
+ WAL is stored on high-latency rotating disks, benefits can be
significant even on storage media with very fast sync times, such as
solid-state drives or RAID arrays with a battery-backed write cache;
but this should definitely be tested against a representative workload.
@@ -828,16 +828,16 @@
<para>
<acronym>WAL</acronym> is automatically enabled; no action is
required from the administrator except ensuring that the
- disk-space requirements for the <acronym>WAL</acronym> logs are met,
+ disk-space requirements for the <acronym>WAL</acronym> files are met,
and that any necessary tuning is done (see <xref
linkend="wal-configuration"/>).
</para>
<para>
<acronym>WAL</acronym> records are appended to the <acronym>WAL</acronym>
- logs as each new record is written. The insert position is described by
+ files as each new record is written. The insert position is described by
a Log Sequence Number (<acronym>LSN</acronym>) that is a byte offset into
- the logs, increasing monotonically with each new record.
+ the WAL, increasing monotonically with each new record.
<acronym>LSN</acronym> values are returned as the datatype
<link linkend="datatype-pg-lsn"><type>pg_lsn</type></link>. Values can be
compared to calculate the volume of <acronym>WAL</acronym> data that
@@ -846,12 +846,12 @@
</para>
<para>
- <acronym>WAL</acronym> logs are stored in the directory
+ <acronym>WAL</acronym> files are stored in the directory
<filename>pg_wal</filename> under the data directory, as a set of
segment files, normally each 16 MB in size (but the size can be changed
by altering the <option>--wal-segsize</option> <application>initdb</application> option). Each segment is
divided into pages, normally 8 kB each (this size can be changed via the
- <option>--with-wal-blocksize</option> configure option). The log record headers
+ <option>--with-wal-blocksize</option> configure option). The WAL record headers
are described in <filename>access/xlogrecord.h</filename>; the record
content is dependent on the type of event that is being logged. Segment
files are given ever-increasing numbers as names, starting at
@@ -861,7 +861,7 @@
</para>
<para>
- It is advantageous if the log is located on a different disk from the
+ It is advantageous if the WAL is located on a different disk from the
main database files. This can be achieved by moving the
<filename>pg_wal</filename> directory to another location (while the server
is shut down, of course) and creating a symbolic link from the
@@ -877,19 +877,19 @@
on the disk. A power failure in such a situation might lead to
irrecoverable data corruption. Administrators should try to ensure
that disks holding <productname>PostgreSQL</productname>'s
- <acronym>WAL</acronym> log files do not make such false reports.
+ <acronym>WAL</acronym> files do not make such false reports.
(See <xref linkend="wal-reliability"/>.)
</para>
<para>
- After a checkpoint has been made and the log flushed, the
+ After a checkpoint has been made and the WAL flushed, the
checkpoint's position is saved in the file
<filename>pg_control</filename>. Therefore, at the start of recovery,
the server first reads <filename>pg_control</filename> and
then the checkpoint record; then it performs the REDO operation by
- scanning forward from the log location indicated in the checkpoint
+ scanning forward from the WAL location indicated in the checkpoint
record. Because the entire content of data pages is saved in the
- log on the first page modification after a checkpoint (assuming
+ WAL on the first page modification after a checkpoint (assuming
<xref linkend="guc-full-page-writes"/> is not disabled), all pages
changed since the checkpoint will be restored to a consistent
state.
@@ -897,7 +897,7 @@
<para>
To deal with the case where <filename>pg_control</filename> is
- corrupt, we should support the possibility of scanning existing log
+ corrupt, we should support the possibility of scanning existing WAL
segments in reverse order — newest to oldest — in order to find the
latest checkpoint. This has not been implemented yet.
<filename>pg_control</filename> is small enough (less than one disk page)
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b4bc06e5f5..6bb37cbecf 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -251,21 +251,21 @@
#archive_mode = off # enables archiving; off, on, or always
# (change requires restart)
-#archive_library = '' # library to use to archive a logfile segment
+#archive_library = '' # library to use to archive a WAL file
# (empty string indicates archive_command should
# be used)
-#archive_command = '' # command to use to archive a logfile segment
+#archive_command = '' # command to use to archive a WAL file
# placeholders: %p = path of file to archive
# %f = file name only
# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
-#archive_timeout = 0 # force a logfile segment switch after this
+#archive_timeout = 0 # force a WAL file switch after this
# number of seconds; 0 disables
# - Archive Recovery -
# These are only used in recovery mode.
-#restore_command = '' # command to use to restore an archived logfile segment
+#restore_command = '' # command to use to restore an archived WAL file
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
--
2.25.1
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-20 01:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 04:32 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-20 07:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 11:55 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
@ 2022-07-21 04:20 ` Kyotaro Horiguchi <[email protected]>
2022-07-23 09:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 137+ messages in thread
From: Kyotaro Horiguchi @ 2022-07-21 04:20 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]
At Wed, 20 Jul 2022 17:25:33 +0530, Bharath Rupireddy <[email protected]> wrote in
> On Wed, Jul 20, 2022 at 12:55 PM Kyotaro Horiguchi
> <[email protected]> wrote:
> PSA v7 patch set.
Thanks. Looks perfect, but (sorry..) in the final checking, I found
"log archive" in the doc. If you agree to it please merge the
attached (or refined one) and I'd call it a day.
regards.
--
Kyotaro Horiguchi
NTT Open Source Software Center
Attachments:
[text/x-patch] fix_log_WAL.patch (1.4K, ../../[email protected]/2-fix_log_WAL.patch)
download | inline diff:
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index c0b89a3c01..e5344eb277 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2659,7 +2659,7 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
<listitem>
<para>
If set to true, the backup will wait until the last required WAL
- segment has been archived, or emit a warning if log archiving is
+ segment has been archived, or emit a warning if WAL archiving is
not enabled. If false, the backup will neither wait nor warn,
leaving the client responsible for ensuring the required log is
available. The default is true.
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 56ac7b754b..e50f00afa8 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -318,7 +318,7 @@ PostgreSQL documentation
backup. This will include all write-ahead logs generated during
the backup. Unless the method <literal>none</literal> is specified,
it is possible to start a postmaster in the target
- directory without the need to consult the log archive, thus
+ directory without the need to consult the WAL archive, thus
making the output a completely standalone backup.
</para>
<para>
^ permalink raw reply [nested|flat] 137+ messages in thread
* Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-20 01:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 04:32 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-20 07:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 11:55 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-21 04:20 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
@ 2022-07-23 09:58 ` Bharath Rupireddy <[email protected]>
0 siblings, 0 replies; 137+ messages in thread
From: Bharath Rupireddy @ 2022-07-23 09:58 UTC (permalink / raw)
To: Kyotaro Horiguchi <[email protected]>; +Cc: Nathan Bossart <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, Jul 21, 2022 at 9:50 AM Kyotaro Horiguchi
<[email protected]> wrote:
>
> At Wed, 20 Jul 2022 17:25:33 +0530, Bharath Rupireddy <[email protected]> wrote in
> > On Wed, Jul 20, 2022 at 12:55 PM Kyotaro Horiguchi
> > <[email protected]> wrote:
> > PSA v7 patch set.
>
> Thanks. Looks perfect, but (sorry..) in the final checking, I found
> "log archive" in the doc. If you agree to it please merge the
> attached (or refined one) and I'd call it a day.
Merged. PSA v8 patch set.
Regards,
Bharath Rupireddy.
Attachments:
[application/octet-stream] v8-0001-Use-WAL-segment-instead-of-log-segment.patch (9.0K, ../../CALj2ACVKmOP145_6wWCYAPU019tTEio8Gwxe7aicyJGy516zug@mail.gmail.com/2-v8-0001-Use-WAL-segment-instead-of-log-segment.patch)
download | inline diff:
From ec8879007aac9a5b81abf6c627acc7d65c178944 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Sat, 23 Jul 2022 09:36:56 +0000
Subject: [PATCH v8] Use "WAL segment" instead of "log segment"
We are using "log segment" in various user-facing messages, the
term "log" can mean server logs as well. The "WAL segment" suits
well here and it is consistently used across the other user-facing
messages.
Author: Bharath Rupireddy
---
src/backend/access/transam/xlogreader.c | 10 +++++-----
src/backend/access/transam/xlogrecovery.c | 6 +++---
src/backend/access/transam/xlogutils.c | 4 ++--
src/backend/replication/walreceiver.c | 6 +++---
src/bin/pg_resetwal/pg_resetwal.c | 2 +-
src/bin/pg_upgrade/controldata.c | 2 +-
src/bin/pg_waldump/pg_waldump.c | 4 ++--
7 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index f3dc4b7797..58f1a32b00 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -1209,7 +1209,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid magic number %04X in log segment %s, offset %u",
+ "invalid magic number %04X in WAL segment %s, offset %u",
hdr->xlp_magic,
fname,
offset);
@@ -1223,7 +1223,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1264,7 +1264,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
/* hmm, first page of file doesn't have a long header? */
report_invalid_record(state,
- "invalid info bits %04X in log segment %s, offset %u",
+ "invalid info bits %04X in WAL segment %s, offset %u",
hdr->xlp_info,
fname,
offset);
@@ -1283,7 +1283,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "unexpected pageaddr %X/%X in log segment %s, offset %u",
+ "unexpected pageaddr %X/%X in WAL segment %s, offset %u",
LSN_FORMAT_ARGS(hdr->xlp_pageaddr),
fname,
offset);
@@ -1308,7 +1308,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
XLogFileName(fname, state->seg.ws_tli, segno, state->segcxt.ws_segsize);
report_invalid_record(state,
- "out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u",
+ "out-of-sequence timeline ID %u (after %u) in WAL segment %s, offset %u",
hdr->xlp_tli,
state->latestPageTLI,
fname,
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 5d6f1b5e46..306a9f40e9 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -3018,7 +3018,7 @@ ReadRecord(XLogPrefetcher *xlogprefetcher, int emode,
XLogFileName(fname, xlogreader->seg.ws_tli, segno,
wal_segment_size);
ereport(emode_for_corrupt_record(emode, xlogreader->EndRecPtr),
- (errmsg("unexpected timeline ID %u in log segment %s, offset %u",
+ (errmsg("unexpected timeline ID %u in WAL segment %s, offset %u",
xlogreader->latestPageTLI,
fname,
offset)));
@@ -3223,13 +3223,13 @@ retry:
errno = save_errno;
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %u: %m",
+ errmsg("could not read from WAL segment %s, offset %u: %m",
fname, readOff)));
}
else
ereport(emode_for_corrupt_record(emode, targetPagePtr + reqLen),
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %u: read %d of %zu",
+ errmsg("could not read from WAL segment %s, offset %u: read %d of %zu",
fname, readOff, r, (Size) XLOG_BLCKSZ)));
goto next_record_is_invalid;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 0cda22597f..9e3a000768 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -1049,14 +1049,14 @@ WALReadRaiseError(WALReadError *errinfo)
errno = errinfo->wre_errno;
ereport(ERROR,
(errcode_for_file_access(),
- errmsg("could not read from log segment %s, offset %d: %m",
+ errmsg("could not read from WAL segment %s, offset %d: %m",
fname, errinfo->wre_off)));
}
else if (errinfo->wre_read == 0)
{
ereport(ERROR,
(errcode(ERRCODE_DATA_CORRUPTED),
- errmsg("could not read from log segment %s, offset %d: read %d of %d",
+ errmsg("could not read from WAL segment %s, offset %d: read %d of %d",
fname, errinfo->wre_off, errinfo->wre_read,
errinfo->wre_req)));
}
diff --git a/src/backend/replication/walreceiver.c b/src/backend/replication/walreceiver.c
index 3d37c1fe62..3767466ef3 100644
--- a/src/backend/replication/walreceiver.c
+++ b/src/backend/replication/walreceiver.c
@@ -616,7 +616,7 @@ WalReceiverMain(void)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
@@ -930,7 +930,7 @@ XLogWalRcvWrite(char *buf, Size nbytes, XLogRecPtr recptr, TimeLineID tli)
errno = save_errno;
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not write to log segment %s "
+ errmsg("could not write to WAL segment %s "
"at offset %u, length %lu: %m",
xlogfname, startoff, (unsigned long) segbytes)));
}
@@ -1042,7 +1042,7 @@ XLogWalRcvClose(XLogRecPtr recptr, TimeLineID tli)
if (close(recvFile) != 0)
ereport(PANIC,
(errcode_for_file_access(),
- errmsg("could not close log segment %s: %m",
+ errmsg("could not close WAL segment %s: %m",
xlogfname)));
/*
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index d4772a2965..7adf79eeed 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -788,7 +788,7 @@ PrintNewControlValues(void)
XLogFileName(fname, ControlFile.checkPointCopy.ThisTimeLineID,
newXlogSegNo, WalSegSz);
- printf(_("First log segment after reset: %s\n"), fname);
+ printf(_("First WAL segment after reset: %s\n"), fname);
if (set_mxid != 0)
{
diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c
index 07de918358..678e8ebf6b 100644
--- a/src/bin/pg_upgrade/controldata.c
+++ b/src/bin/pg_upgrade/controldata.c
@@ -350,7 +350,7 @@ get_control_data(ClusterInfo *cluster, bool live_check)
cluster->controldata.chkpnt_nxtmxoff = str2uint(p);
got_mxoff = true;
}
- else if ((p = strstr(bufin, "First log segment after reset:")) != NULL)
+ else if ((p = strstr(bufin, "First WAL segment after reset:")) != NULL)
{
/* Skip the colon and any whitespace after it */
p = strchr(p, ':');
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6528113628..4eebeadc8c 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -667,7 +667,7 @@ usage(void)
printf(_(" -F, --fork=FORK only show records that modify blocks in fork FORK;\n"
" valid names are main, fsm, vm, init\n"));
printf(_(" -n, --limit=N number of records to display\n"));
- printf(_(" -p, --path=PATH directory in which to find log segment files or a\n"
+ printf(_(" -p, --path=PATH directory in which to find WAL segment files or a\n"
" directory with a ./pg_wal that contains such files\n"
" (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n"));
printf(_(" -q, --quiet do not print any output, except for errors\n"));
@@ -675,7 +675,7 @@ usage(void)
" use --rmgr=list to list valid resource manager names\n"));
printf(_(" -R, --relation=T/D/R only show records that modify blocks in relation T/D/R\n"));
printf(_(" -s, --start=RECPTR start reading at WAL location RECPTR\n"));
- printf(_(" -t, --timeline=TLI timeline from which to read log records\n"
+ printf(_(" -t, --timeline=TLI timeline from which to read WAL records\n"
" (default: 1 or the value used in STARTSEG)\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -w, --fullpage only show records with a full page write\n"));
--
2.34.1
[application/octet-stream] v8-0002-Consistently-use-WAL-file-s-in-docs.patch (21.5K, ../../CALj2ACVKmOP145_6wWCYAPU019tTEio8Gwxe7aicyJGy516zug@mail.gmail.com/3-v8-0002-Consistently-use-WAL-file-s-in-docs.patch)
download | inline diff:
From e52b42062e04d026d7efcd9d8bebf03fcf77e8b2 Mon Sep 17 00:00:00 2001
From: Bharath Rupireddy <[email protected]>
Date: Sat, 23 Jul 2022 09:55:57 +0000
Subject: [PATCH v8] Consistently use "WAL file(s)" in docs
Authors: Kyotaro Horiguchi, Bharath Rupireddy
---
doc/src/sgml/backup.sgml | 14 ++---
doc/src/sgml/config.sgml | 4 +-
doc/src/sgml/protocol.sgml | 2 +-
doc/src/sgml/ref/pg_basebackup.sgml | 2 +-
doc/src/sgml/ref/pg_waldump.sgml | 10 ++--
doc/src/sgml/wal.sgml | 60 +++++++++----------
src/backend/utils/misc/postgresql.conf.sample | 8 +--
7 files changed, 50 insertions(+), 50 deletions(-)
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 73a774d3d7..cc5ae59ac2 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -1095,7 +1095,7 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
require that you have enough free space on your system to hold two
copies of your existing database. If you do not have enough space,
you should at least save the contents of the cluster's <filename>pg_wal</filename>
- subdirectory, as it might contain logs which
+ subdirectory, as it might contain WAL files which
were not archived before the system went down.
</para>
</listitem>
@@ -1173,8 +1173,8 @@ SELECT * FROM pg_backup_stop(wait_for_archive => true);
which tells <productname>PostgreSQL</productname> how to retrieve archived
WAL file segments. Like the <varname>archive_command</varname>, this is
a shell command string. It can contain <literal>%f</literal>, which is
- replaced by the name of the desired log file, and <literal>%p</literal>,
- which is replaced by the path name to copy the log file to.
+ replaced by the name of the desired WAL file, and <literal>%p</literal>,
+ which is replaced by the path name to copy the WAL file to.
(The path name is relative to the current working directory,
i.e., the cluster's data directory.)
Write <literal>%%</literal> if you need to embed an actual <literal>%</literal>
@@ -1462,9 +1462,9 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
<link linkend="sql-createtablespace"><command>CREATE TABLESPACE</command></link>
commands are WAL-logged with the literal absolute path, and will
therefore be replayed as tablespace creations with the same
- absolute path. This might be undesirable if the log is being
+ absolute path. This might be undesirable if the WAL is being
replayed on a different machine. It can be dangerous even if the
- log is being replayed on the same machine, but into a new data
+ WAL is being replayed on the same machine, but into a new data
directory: the replay will still overwrite the contents of the
original tablespace. To avoid potential gotchas of this sort,
the best practice is to take a new base backup after creating or
@@ -1481,11 +1481,11 @@ archive_command = 'local_backup_script.sh "%p" "%f"'
we might need to fix partially-written disk pages. Depending on
your system hardware and software, the risk of partial writes might
be small enough to ignore, in which case you can significantly
- reduce the total volume of archived logs by turning off page
+ reduce the total volume of archived WAL files by turning off page
snapshots using the <xref linkend="guc-full-page-writes"/>
parameter. (Read the notes and warnings in <xref linkend="wal"/>
before you do so.) Turning off page snapshots does not prevent
- use of the logs for PITR operations. An area for future
+ use of the WAL for PITR operations. An area for future
development is to compress archived WAL data by removing
unnecessary page copies even when <varname>full_page_writes</varname> is
on. In the meantime, administrators might wish to reduce the number
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index e2d728e0c4..4e92c0c5eb 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4228,7 +4228,7 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows
</term>
<listitem>
<para>
- Specifies the minimum size of past log file segments kept in the
+ Specifies the minimum size of past WAL files kept in the
<filename>pg_wal</filename>
directory, in case a standby server needs to fetch them for streaming
replication. If a standby
@@ -4821,7 +4821,7 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
needs to control the amount of time to wait for new WAL data to be
available. For example, in archive recovery, it is possible to
make the recovery more responsive in the detection of a new WAL
- log file by reducing the value of this parameter. On a system with
+ file by reducing the value of this parameter. On a system with
low WAL activity, increasing it reduces the amount of requests necessary
to access WAL archives, something useful for example in cloud
environments where the number of times an infrastructure is accessed
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index c0b89a3c01..e5344eb277 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2659,7 +2659,7 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
<listitem>
<para>
If set to true, the backup will wait until the last required WAL
- segment has been archived, or emit a warning if log archiving is
+ segment has been archived, or emit a warning if WAL archiving is
not enabled. If false, the backup will neither wait nor warn,
leaving the client responsible for ensuring the required log is
available. The default is true.
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 56ac7b754b..e50f00afa8 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -318,7 +318,7 @@ PostgreSQL documentation
backup. This will include all write-ahead logs generated during
the backup. Unless the method <literal>none</literal> is specified,
it is possible to start a postmaster in the target
- directory without the need to consult the log archive, thus
+ directory without the need to consult the WAL archive, thus
making the output a completely standalone backup.
</para>
<para>
diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml
index 57746d9421..2e2166bb6f 100644
--- a/doc/src/sgml/ref/pg_waldump.sgml
+++ b/doc/src/sgml/ref/pg_waldump.sgml
@@ -53,7 +53,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">startseg</replaceable></term>
<listitem>
<para>
- Start reading at the specified log segment file. This implicitly determines
+ Start reading at the specified WAL segment file. This implicitly determines
the path in which files will be searched for, and the timeline to use.
</para>
</listitem>
@@ -63,7 +63,7 @@ PostgreSQL documentation
<term><replaceable class="parameter">endseg</replaceable></term>
<listitem>
<para>
- Stop after reading the specified log segment file.
+ Stop after reading the specified WAL segment file.
</para>
</listitem>
</varlistentry>
@@ -141,7 +141,7 @@ PostgreSQL documentation
<term><option>--path=<replaceable>path</replaceable></option></term>
<listitem>
<para>
- Specifies a directory to search for log segment files or a
+ Specifies a directory to search for WAL segment files or a
directory with a <literal>pg_wal</literal> subdirectory that
contains such files. The default is to search in the current
directory, the <literal>pg_wal</literal> subdirectory of the
@@ -203,7 +203,7 @@ PostgreSQL documentation
<listitem>
<para>
WAL location at which to start reading. The default is to start reading
- the first valid log record found in the earliest file found.
+ the first valid WAL record found in the earliest file found.
</para>
</listitem>
</varlistentry>
@@ -213,7 +213,7 @@ PostgreSQL documentation
<term><option>--timeline=<replaceable>timeline</replaceable></option></term>
<listitem>
<para>
- Timeline from which to read log records. The default is to use the
+ Timeline from which to read WAL records. The default is to use the
value in <replaceable>startseg</replaceable>, if that is specified; otherwise, the
default is 1.
</para>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 01f7379ebb..30842c0396 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -297,12 +297,12 @@
transaction processing. Briefly, <acronym>WAL</acronym>'s central
concept is that changes to data files (where tables and indexes
reside) must be written only after those changes have been logged,
- that is, after log records describing the changes have been flushed
+ that is, after WAL records describing the changes have been flushed
to permanent storage. If we follow this procedure, we do not need
to flush data pages to disk on every transaction commit, because we
know that in the event of a crash we will be able to recover the
database using the log: any changes that have not been applied to
- the data pages can be redone from the log records. (This is
+ the data pages can be redone from the WAL records. (This is
roll-forward recovery, also known as REDO.)
</para>
@@ -323,15 +323,15 @@
<para>
Using <acronym>WAL</acronym> results in a
- significantly reduced number of disk writes, because only the log
+ significantly reduced number of disk writes, because only the WAL
file needs to be flushed to disk to guarantee that a transaction is
committed, rather than every data file changed by the transaction.
- The log file is written sequentially,
- and so the cost of syncing the log is much less than the cost of
+ The WAL file is written sequentially,
+ and so the cost of syncing the WAL is much less than the cost of
flushing the data pages. This is especially true for servers
handling many small transactions touching different parts of the data
store. Furthermore, when the server is processing many small concurrent
- transactions, one <function>fsync</function> of the log file may
+ transactions, one <function>fsync</function> of the WAL file may
suffice to commit many transactions.
</para>
@@ -341,10 +341,10 @@
linkend="continuous-archiving"/>. By archiving the WAL data we can support
reverting to any time instant covered by the available WAL data:
we simply install a prior physical backup of the database, and
- replay the WAL log just as far as the desired time. What's more,
+ replay the WAL just as far as the desired time. What's more,
the physical backup doesn't have to be an instantaneous snapshot
of the database state — if it is made over some period of time,
- then replaying the WAL log for that period will fix any internal
+ then replaying the WAL for that period will fix any internal
inconsistencies.
</para>
</sect1>
@@ -497,15 +497,15 @@
that the heap and index data files have been updated with all
information written before that checkpoint. At checkpoint time, all
dirty data pages are flushed to disk and a special checkpoint record is
- written to the log file. (The change records were previously flushed
+ written to the WAL file. (The change records were previously flushed
to the <acronym>WAL</acronym> files.)
In the event of a crash, the crash recovery procedure looks at the latest
- checkpoint record to determine the point in the log (known as the redo
+ checkpoint record to determine the point in the WAL (known as the redo
record) from which it should start the REDO operation. Any changes made to
data files before that point are guaranteed to be already on disk.
- Hence, after a checkpoint, log segments preceding the one containing
+ Hence, after a checkpoint, WAL segments preceding the one containing
the redo record are no longer needed and can be recycled or removed. (When
- <acronym>WAL</acronym> archiving is being done, the log segments must be
+ <acronym>WAL</acronym> archiving is being done, the WAL segments must be
archived before being recycled or removed.)
</para>
@@ -544,7 +544,7 @@
another factor to consider. To ensure data page consistency,
the first modification of a data page after each checkpoint results in
logging the entire page content. In that case,
- a smaller checkpoint interval increases the volume of output to the WAL log,
+ a smaller checkpoint interval increases the volume of output to the WAL,
partially negating the goal of using a smaller interval,
and in any case causing more disk I/O.
</para>
@@ -614,10 +614,10 @@
<para>
The number of WAL segment files in <filename>pg_wal</filename> directory depends on
<varname>min_wal_size</varname>, <varname>max_wal_size</varname> and
- the amount of WAL generated in previous checkpoint cycles. When old log
+ the amount of WAL generated in previous checkpoint cycles. When old WAL
segment files are no longer needed, they are removed or recycled (that is,
renamed to become future segments in the numbered sequence). If, due to a
- short-term peak of log output rate, <varname>max_wal_size</varname> is
+ short-term peak of WAL output rate, <varname>max_wal_size</varname> is
exceeded, the unneeded segment files will be removed until the system
gets back under this limit. Below that limit, the system recycles enough
WAL files to cover the estimated need until the next checkpoint, and
@@ -650,7 +650,7 @@
which are similar to checkpoints in normal operation: the server forces
all its state to disk, updates the <filename>pg_control</filename> file to
indicate that the already-processed WAL data need not be scanned again,
- and then recycles any old log segment files in the <filename>pg_wal</filename>
+ and then recycles any old WAL segment files in the <filename>pg_wal</filename>
directory.
Restartpoints can't be performed more frequently than checkpoints on the
primary because restartpoints can only be performed at checkpoint records.
@@ -676,12 +676,12 @@
insertion) at a time when an exclusive lock is held on affected
data pages, so the operation needs to be as fast as possible. What
is worse, writing <acronym>WAL</acronym> buffers might also force the
- creation of a new log segment, which takes even more
+ creation of a new WAL segment, which takes even more
time. Normally, <acronym>WAL</acronym> buffers should be written
and flushed by an <function>XLogFlush</function> request, which is
made, for the most part, at transaction commit time to ensure that
transaction records are flushed to permanent storage. On systems
- with high log output, <function>XLogFlush</function> requests might
+ with high WAL output, <function>XLogFlush</function> requests might
not occur often enough to prevent <function>XLogInsertRecord</function>
from having to do writes. On such systems
one should increase the number of <acronym>WAL</acronym> buffers by
@@ -724,7 +724,7 @@
<varname>commit_delay</varname>, so this value is recommended as the
starting point to use when optimizing for a particular workload. While
tuning <varname>commit_delay</varname> is particularly useful when the
- WAL log is stored on high-latency rotating disks, benefits can be
+ WAL is stored on high-latency rotating disks, benefits can be
significant even on storage media with very fast sync times, such as
solid-state drives or RAID arrays with a battery-backed write cache;
but this should definitely be tested against a representative workload.
@@ -828,16 +828,16 @@
<para>
<acronym>WAL</acronym> is automatically enabled; no action is
required from the administrator except ensuring that the
- disk-space requirements for the <acronym>WAL</acronym> logs are met,
+ disk-space requirements for the <acronym>WAL</acronym> files are met,
and that any necessary tuning is done (see <xref
linkend="wal-configuration"/>).
</para>
<para>
<acronym>WAL</acronym> records are appended to the <acronym>WAL</acronym>
- logs as each new record is written. The insert position is described by
+ files as each new record is written. The insert position is described by
a Log Sequence Number (<acronym>LSN</acronym>) that is a byte offset into
- the logs, increasing monotonically with each new record.
+ the WAL, increasing monotonically with each new record.
<acronym>LSN</acronym> values are returned as the datatype
<link linkend="datatype-pg-lsn"><type>pg_lsn</type></link>. Values can be
compared to calculate the volume of <acronym>WAL</acronym> data that
@@ -846,12 +846,12 @@
</para>
<para>
- <acronym>WAL</acronym> logs are stored in the directory
+ <acronym>WAL</acronym> files are stored in the directory
<filename>pg_wal</filename> under the data directory, as a set of
segment files, normally each 16 MB in size (but the size can be changed
by altering the <option>--wal-segsize</option> <application>initdb</application> option). Each segment is
divided into pages, normally 8 kB each (this size can be changed via the
- <option>--with-wal-blocksize</option> configure option). The log record headers
+ <option>--with-wal-blocksize</option> configure option). The WAL record headers
are described in <filename>access/xlogrecord.h</filename>; the record
content is dependent on the type of event that is being logged. Segment
files are given ever-increasing numbers as names, starting at
@@ -861,7 +861,7 @@
</para>
<para>
- It is advantageous if the log is located on a different disk from the
+ It is advantageous if the WAL is located on a different disk from the
main database files. This can be achieved by moving the
<filename>pg_wal</filename> directory to another location (while the server
is shut down, of course) and creating a symbolic link from the
@@ -877,19 +877,19 @@
on the disk. A power failure in such a situation might lead to
irrecoverable data corruption. Administrators should try to ensure
that disks holding <productname>PostgreSQL</productname>'s
- <acronym>WAL</acronym> log files do not make such false reports.
+ <acronym>WAL</acronym> files do not make such false reports.
(See <xref linkend="wal-reliability"/>.)
</para>
<para>
- After a checkpoint has been made and the log flushed, the
+ After a checkpoint has been made and the WAL flushed, the
checkpoint's position is saved in the file
<filename>pg_control</filename>. Therefore, at the start of recovery,
the server first reads <filename>pg_control</filename> and
then the checkpoint record; then it performs the REDO operation by
- scanning forward from the log location indicated in the checkpoint
+ scanning forward from the WAL location indicated in the checkpoint
record. Because the entire content of data pages is saved in the
- log on the first page modification after a checkpoint (assuming
+ WAL on the first page modification after a checkpoint (assuming
<xref linkend="guc-full-page-writes"/> is not disabled), all pages
changed since the checkpoint will be restored to a consistent
state.
@@ -897,7 +897,7 @@
<para>
To deal with the case where <filename>pg_control</filename> is
- corrupt, we should support the possibility of scanning existing log
+ corrupt, we should support the possibility of scanning existing WAL
segments in reverse order — newest to oldest — in order to find the
latest checkpoint. This has not been implemented yet.
<filename>pg_control</filename> is small enough (less than one disk page)
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b4bc06e5f5..6bb37cbecf 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -251,21 +251,21 @@
#archive_mode = off # enables archiving; off, on, or always
# (change requires restart)
-#archive_library = '' # library to use to archive a logfile segment
+#archive_library = '' # library to use to archive a WAL file
# (empty string indicates archive_command should
# be used)
-#archive_command = '' # command to use to archive a logfile segment
+#archive_command = '' # command to use to archive a WAL file
# placeholders: %p = path of file to archive
# %f = file name only
# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
-#archive_timeout = 0 # force a logfile segment switch after this
+#archive_timeout = 0 # force a WAL file switch after this
# number of seconds; 0 disables
# - Archive Recovery -
# These are only used in recovery mode.
-#restore_command = '' # command to use to restore an archived logfile segment
+#restore_command = '' # command to use to restore an archived WAL file
# placeholders: %p = path of file to restore
# %f = file name only
# e.g. 'cp /mnt/server/archivedir/%f %p'
--
2.34.1
^ permalink raw reply [nested|flat] 137+ messages in thread
end of thread, other threads:[~2022-07-23 09:58 UTC | newest]
Thread overview: 137+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/12] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH v7 5/9] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/12] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: LZ4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 5/8] (re)add wal_compression_method=lz4 Justin Pryzby <[email protected]>
2021-03-12 20:32 [PATCH 05/10] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-05-25 04:32 [PATCH v9 5/9] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2021-05-25 04:32 [PATCH v8 5/9] (re)add wal_compression_method: lz4 Justin Pryzby <[email protected]>
2022-04-01 14:24 Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-04-25 19:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-18 10:54 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-18 18:30 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-19 09:13 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-19 16:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Nathan Bossart <[email protected]>
2022-07-20 01:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 04:32 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-20 07:25 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-20 11:55 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
2022-07-21 04:20 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Kyotaro Horiguchi <[email protected]>
2022-07-23 09:58 ` Re: Use "WAL segment" instead of "log segment" consistently in user-facing messages Bharath Rupireddy <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox