($INBOX_DIR/description missing)  

help / color / mirror / Atom feed

[PATCH 05/12] (re)add wal_compression_method: lz4
134+ messages / 8 participants
[nested] [flat]

* [PATCH 05/12] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 doc/src/sgml/config.sgml                      |  3 ++-
 src/backend/access/transam/xlog.c             |  3 +++
 src/backend/access/transam/xloginsert.c       | 17 ++++++++++++++++-
 src/backend/access/transam/xlogreader.c       | 12 ++++++++++++
 src/backend/utils/misc/postgresql.conf.sample |  2 +-
 src/include/access/xlog_internal.h            |  1 +
 6 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bed89d055d..1f21e96922 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3ece007922..23f5bdc335 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 530007f55a..2a2bccbe5c 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
-- 
2.17.0


--45Z9DzgjV8m4Oswq
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0006-add-wal_compression_method-zstd.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 src/backend/access/transam/xlog.c       |   3 +
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  12 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 8 files changed, 234 insertions(+)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ = 0,
 	WAL_COMPRESSION_ZLIB = 1,
+	WAL_COMPRESSION_LZ4 = 2,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0005-re-add-wal_compression_method-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 doc/src/sgml/config.sgml                |   3 +-
 src/backend/access/transam/xlog.c       |   5 +-
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  13 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 9 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool		EnableHotStandby = false;
 bool		fullPageWrites = true;
 bool		wal_log_hints = false;
 bool		wal_compression = false;
-int			wal_compression_method = WAL_COMPRESSION_ZLIB;
+int			wal_compression_method = WAL_COMPRESSION_LZ4;
 char	   *wal_consistency_checking_string = NULL;
 bool	   *wal_consistency_checking = NULL;
 bool		wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  17 +-
 src/backend/access/transam/xlogreader.c       |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 241 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/12] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 doc/src/sgml/config.sgml                      |  3 ++-
 src/backend/access/transam/xlog.c             |  3 +++
 src/backend/access/transam/xloginsert.c       | 17 ++++++++++++++++-
 src/backend/access/transam/xlogreader.c       | 12 ++++++++++++
 src/backend/utils/misc/postgresql.conf.sample |  2 +-
 src/include/access/xlog_internal.h            |  1 +
 6 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bed89d055d..1f21e96922 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3ece007922..23f5bdc335 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 530007f55a..2a2bccbe5c 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
-- 
2.17.0


--45Z9DzgjV8m4Oswq
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0006-add-wal_compression_method-zstd.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 src/backend/access/transam/xlog.c       |   3 +
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  12 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 8 files changed, 234 insertions(+)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ = 0,
 	WAL_COMPRESSION_ZLIB = 1,
+	WAL_COMPRESSION_LZ4 = 2,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0005-re-add-wal_compression_method-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 doc/src/sgml/config.sgml                |   3 +-
 src/backend/access/transam/xlog.c       |   5 +-
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  13 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 9 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool		EnableHotStandby = false;
 bool		fullPageWrites = true;
 bool		wal_log_hints = false;
 bool		wal_compression = false;
-int			wal_compression_method = WAL_COMPRESSION_ZLIB;
+int			wal_compression_method = WAL_COMPRESSION_LZ4;
 char	   *wal_consistency_checking_string = NULL;
 bool	   *wal_consistency_checking = NULL;
 bool		wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  17 +-
 src/backend/access/transam/xlogreader.c       |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 241 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 src/backend/access/transam/xlog.c       |   3 +
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  12 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 8 files changed, 234 insertions(+)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ = 0,
 	WAL_COMPRESSION_ZLIB = 1,
+	WAL_COMPRESSION_LZ4 = 2,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0005-re-add-wal_compression_method-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 doc/src/sgml/config.sgml                |   3 +-
 src/backend/access/transam/xlog.c       |   5 +-
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  13 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 9 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool		EnableHotStandby = false;
 bool		fullPageWrites = true;
 bool		wal_log_hints = false;
 bool		wal_compression = false;
-int			wal_compression_method = WAL_COMPRESSION_ZLIB;
+int			wal_compression_method = WAL_COMPRESSION_LZ4;
 char	   *wal_consistency_checking_string = NULL;
 bool	   *wal_consistency_checking = NULL;
 bool		wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  17 +-
 src/backend/access/transam/xlogreader.c       |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 241 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH v7 5/9] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 doc/src/sgml/config.sgml                      |  3 ++-
 doc/src/sgml/install-windows.sgml             |  2 +-
 doc/src/sgml/installation.sgml                |  5 +++--
 src/backend/access/transam/xlog.c             |  3 +++
 src/backend/access/transam/xloginsert.c       | 17 ++++++++++++++++-
 src/backend/access/transam/xlogreader.c       | 12 ++++++++++++
 src/backend/utils/misc/postgresql.conf.sample |  2 +-
 src/include/access/xlog_internal.h            |  1 +
 8 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 70effa6345..f2209f5df0 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3147,7 +3147,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml
index db53ee85a8..a023584722 100644
--- a/doc/src/sgml/install-windows.sgml
+++ b/doc/src/sgml/install-windows.sgml
@@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m";
      <term><productname>LZ4</productname></term>
      <listitem><para>
       Required for supporting <productname>LZ4</productname> compression
-      method for compressing the table data. Binaries and source can be
+      method for compressing table or WAL data. Binaries and source can be
       downloaded from
       <ulink url="https://github.com/lz4/lz4/releases"></ulink;.
      </para></listitem>
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 3c0aa118c7..7c8cdac25a 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -270,7 +270,8 @@ su - postgres
      <para>
       You need <productname>LZ4</productname>, if you want to support
       compression of data with this method; see
-      <xref linkend="guc-default-toast-compression"/>.
+      <xref linkend="guc-default-toast-compression"/> and
+      <xref linkend="guc-wal-compression-method"/>.
      </para>
     </listitem>
 
@@ -980,7 +981,7 @@ build-postgresql:
         <para>
          Build with <productname>LZ4</productname> compression support.
          This allows the use of <productname>LZ4</productname> for
-         compression of table data.
+         compression of table and WAL data.
         </para>
        </listitem>
       </varlistentry>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a43e04fc0b..999896487e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 5ab07621d6..30804e8cd5 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index baed4d9228..6421ddbde8 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -219,7 +219,7 @@
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_init_zero = on			# zero-fill new WAL files
 #wal_recycle = on			# recycle WAL files
 #wal_buffers = -1			# min 32kB, -1 sets based on shared_buffers
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index ceca0f7189..fc57ea728e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
-- 
2.17.0


--JYK4vJDZwFMowpUq
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="v7-0006-add-wal_compression_method-zstd.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 src/backend/access/transam/xlog.c       |   3 +
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  12 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 8 files changed, 234 insertions(+)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ = 0,
 	WAL_COMPRESSION_ZLIB = 1,
+	WAL_COMPRESSION_LZ4 = 2,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0005-re-add-wal_compression_method-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 doc/src/sgml/config.sgml                |   3 +-
 src/backend/access/transam/xlog.c       |   5 +-
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  13 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 9 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool		EnableHotStandby = false;
 bool		fullPageWrites = true;
 bool		wal_log_hints = false;
 bool		wal_compression = false;
-int			wal_compression_method = WAL_COMPRESSION_ZLIB;
+int			wal_compression_method = WAL_COMPRESSION_LZ4;
 char	   *wal_consistency_checking_string = NULL;
 bool	   *wal_consistency_checking = NULL;
 bool		wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  17 +-
 src/backend/access/transam/xlogreader.c       |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 241 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 5/8] (re)add wal_compression_method=lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 src/backend/access/transam/xlog.c       |   3 +
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  12 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 8 files changed, 234 insertions(+)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index ee73bc3afd..38ac5f9e94 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 143df55fcb..920c47e4d7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1607,6 +1613,12 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data, bkpb->bimg_len, BLCKSZ);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b80759ed45..0993de784e 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ = 0,
 	WAL_COMPRESSION_ZLIB = 1,
+	WAL_COMPRESSION_LZ4 = 2,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--f0KYrhQ4vYSV2aJu
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..f33355e44a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0005-re-add-wal_compression_method-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: LZ4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                               | 176 ++++++++++++++++++++++++
 configure.ac                            |  26 ++++
 doc/src/sgml/config.sgml                |   3 +-
 src/backend/access/transam/xlog.c       |   5 +-
 src/backend/access/transam/xloginsert.c |  12 ++
 src/backend/access/transam/xlogreader.c |  13 ++
 src/include/access/xlog_internal.h      |   1 +
 src/include/pg_config.h.in              |   3 +
 src/tools/msvc/Solution.pm              |   1 +
 9 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index fad817bb38..9b590eb432 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13378,6 +13518,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 0ed53571dd..2b3b5676eb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1407,6 +1422,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 8bae73b4ec..609d954a3a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -99,7 +99,7 @@ bool		EnableHotStandby = false;
 bool		fullPageWrites = true;
 bool		wal_log_hints = false;
 bool		wal_compression = false;
-int			wal_compression_method = WAL_COMPRESSION_ZLIB;
+int			wal_compression_method = WAL_COMPRESSION_LZ4;
 char	   *wal_consistency_checking_string = NULL;
 bool	   *wal_consistency_checking = NULL;
 bool		wal_init_zero = true;
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--0qVF/w3MHQqLSynd
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  12 ++
 src/backend/access/transam/xlogreader.c       |  13 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 238 insertions(+), 2 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 34e1227381..9fe5c30236 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(BLCKSZ)
 
@@ -884,6 +888,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, PGLZ_MAX_BLCKSZ, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index afca22a26c..fa1e38a810 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1558,6 +1562,8 @@ wal_compression_name(WalCompression compression)
 			return "pglz";
 		case WAL_COMPRESSION_ZLIB:
 			return "zlib";
+		case WAL_COMPRESSION_LZ4:
+			return "lz4";
 		default:
 			return "???";
 	}
@@ -1608,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index d653839b97..e70886b81c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -328,6 +328,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--XsQoSWH+UP9D9v3l
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread

* [PATCH 05/10] (re)add wal_compression_method: lz4
@ 2021-03-12 20:32  Justin Pryzby <[email protected]>
  0 siblings, 0 replies; 134+ messages in thread

From: Justin Pryzby @ 2021-03-12 20:32 UTC (permalink / raw)

---
 configure                                     | 176 ++++++++++++++++++
 configure.ac                                  |  26 +++
 doc/src/sgml/config.sgml                      |   3 +-
 src/backend/access/transam/xlog.c             |   3 +
 src/backend/access/transam/xloginsert.c       |  17 +-
 src/backend/access/transam/xlogreader.c       |  12 ++
 src/backend/utils/misc/postgresql.conf.sample |   2 +-
 src/include/access/xlog_internal.h            |   1 +
 src/include/pg_config.h.in                    |   3 +
 src/tools/msvc/Solution.pm                    |   1 +
 10 files changed, 241 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 3fd4cecbeb..fed440adcf 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
 LD
 LDFLAGS_SL
 LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
 with_zlib
 with_system_tzdata
 with_libxslt
@@ -864,6 +867,7 @@ with_libxml
 with_libxslt
 with_system_tzdata
 with_zlib
+with_lz4
 with_gnu_ld
 with_ssl
 with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
 XML2_CONFIG
 XML2_CFLAGS
 XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
 LDFLAGS_EX
 LDFLAGS_SL
 PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
   --with-system-tzdata=DIR
                           use system time zone data in DIR
   --without-zlib          do not use Zlib
+  --with-lz4              build with LZ4 support
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
   --with-ssl=LIB          use LIB for SSL/TLS support (openssl)
   --with-openssl          obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
   XML2_CONFIG path to xml2-config utility
   XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
   XML2_LIBS   linker flags for XML2, overriding pkg-config
+  LZ4_CFLAGS  C compiler flags for LZ4, overriding pkg-config
+  LZ4_LIBS    linker flags for LZ4, overriding pkg-config
   LDFLAGS_EX  extra linker flags for linking executables only
   LDFLAGS_SL  extra linker flags for linking shared libraries only
   PERL        Perl program
@@ -8563,6 +8572,137 @@ fi
 
 
 
+#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+  withval=$with_lz4;
+  case $withval in
+    yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+    pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+    pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+        else
+	        LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LZ4_PKG_ERRORS" >&5
+
+	as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+	{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/;.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+	LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+	LZ4_LIBS=$pkg_cv_LZ4_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -13381,6 +13521,42 @@ fi
 
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  for ac_header in lz4/lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+  for ac_header in lz4.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+  as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   for ac_header in gssapi/gssapi.h
 do :
diff --git a/configure.ac b/configure.ac
index 2f1585adc0..8c454128bb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -986,6 +986,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
               [do not use Zlib])
 AC_SUBST(with_zlib)
 
+#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+              [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+  PKG_CHECK_MODULES(LZ4, liblz4)
+  LIBS="$LZ4_LIBS $LIBS"
+  CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
 #
 # Assignments
 #
@@ -1410,6 +1425,17 @@ failure.  It is possible the compiler isn't looking in the proper directory.
 Use --without-zlib to disable zlib support.])])
 fi
 
+if test "$with_lz4" = yes; then
+  ac_save_CPPFLAGS=$CPPFLAGS
+  CPPFLAGS="$LZ4_CFLAGS $CPPFLAGS"
+
+  # Verify we have LZ4's header files
+  AC_CHECK_HEADERS(lz4/lz4.h, [],
+       [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+
+  CPPFLAGS=$ac_save_CPPFLAGS
+fi
+
 if test "$with_gssapi" = yes ; then
   AC_CHECK_HEADERS(gssapi/gssapi.h, [],
 	[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7fb2a84626..257775c83b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3082,7 +3082,8 @@ include_dir 'conf.d'
        <para>
         This parameter selects the compression method used to compress WAL when
         <varname>wal_compression</varname> is enabled.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>pglz</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index f5d4450654..984ce39cc7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -186,6 +186,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"pglz", WAL_COMPRESSION_PGLZ, false},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index a93b33464f..3c15286bd8 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -891,6 +898,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 0d8830fc50..97165f1bb1 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -37,6 +37,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -1544,6 +1548,7 @@ XLogRecGetBlockData(XLogReaderState *record, uint8 block_id, Size *len)
 struct walcompression walmethods[] = {
 	{"pglz",	WAL_COMPRESSION_PGLZ},
 	{"zlib",	WAL_COMPRESSION_ZLIB},
+	{"lz4",		WAL_COMPRESSION_LZ4},
 };
 
 /*
@@ -1609,6 +1614,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index ef69a94492..76f494cb9b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -213,7 +213,7 @@
 					#   open_sync
 #full_page_writes = on			# recover from partial page writes
 #wal_compression = off			# enable compression of full-page writes
-#wal_compression_method = pglz		# pglz, zlib
+#wal_compression_method = pglz		# pglz, zlib, lz4
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
 #wal_init_zero = on			# zero-fill new WAL files
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index b000a21557..b908fa48de 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -336,6 +336,7 @@ typedef enum WalCompression
 {
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21d8d..0a6422da4f 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
 /* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
 #undef USE_LLVM
 
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
 /* Define to select named POSIX semaphores. */
 #undef USE_NAMED_POSIX_SEMAPHORES
 
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4bdb..14605371bb 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
 		USE_ICU => $self->{options}->{icu} ? 1 : undef,
 		USE_LIBXML                 => undef,
 		USE_LIBXSLT                => undef,
+		USE_LZ4                    => undef,
 		USE_LDAP                   => $self->{options}->{ldap} ? 1 : undef,
 		USE_LLVM                   => undef,
 		USE_NAMED_POSIX_SEMAPHORES => undef,
-- 
2.17.0


--jozmn01XJZjDjM3N
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="0006-Default-to-LZ4.patch"



^ permalink  raw  reply  [nested|flat] 134+ messages in thread