Subject: [PATCH v8 5/9] (re)add wal_compression_method: lz4

From: Justin Pryzby <[email-masked]>
Date: Mon, 24 May 2021 23:32:30 -0500

---
 doc/src/sgml/config.sgml                      |  3 ++-
 doc/src/sgml/install-windows.sgml             |  2 +-
 doc/src/sgml/installation.sgml                |  5 +++--
 src/backend/access/transam/xloginsert.c       | 17 ++++++++++++++++-
 src/backend/access/transam/xlogreader.c       | 14 ++++++++++++++
 src/backend/utils/misc/postgresql.conf.sample |  2 +-
 src/include/access/xlog_internal.h            |  1 +
 src/include/access/xlogrecord.h               |  2 +-
 8 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 218a98cfc6..352b31fa81 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3126,7 +3126,8 @@ include_dir 'conf.d'
         server compresses full page images written to WAL when
         <xref linkend="guc-full-page-writes"/> is on or during a base backup.
         A compressed page image will be decompressed during WAL replay.
-        The supported methods are pglz and zlib.
+        The supported methods are pglz, zlib, and (if configured when
+        <productname>PostgreSQL</productname> was built) lz4.
         The default value is <literal>off</literal>.
         Only superusers can change this setting.
        </para>
diff --git a/doc/src/sgml/install-windows.sgml b/doc/src/sgml/install-windows.sgml
index db53ee85a8..a023584722 100644
--- a/doc/src/sgml/install-windows.sgml
+++ b/doc/src/sgml/install-windows.sgml
@@ -299,7 +299,7 @@ $ENV{MSBFLAGS}="/m";
      <term><productname>LZ4</productname></term>
      <listitem><para>
       Required for supporting <productname>LZ4</productname> compression
-      method for compressing the table data. Binaries and source can be
+      method for compressing table or WAL data. Binaries and source can be
       downloaded from
       <ulink url="https://github.com/lz4/lz4/releases"></ulink>.
      </para></listitem>
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 073d5089f7..c7673a4dc8 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -270,7 +270,8 @@ su - postgres
      <para>
       You need <productname>LZ4</productname>, if you want to support
       compression of data with this method; see
-      <xref linkend="guc-default-toast-compression"/>.
+      <xref linkend="guc-default-toast-compression"/> and
+      <xref linkend="guc-wal-compression"/>.
      </para>
     </listitem>
 
@@ -980,7 +981,7 @@ build-postgresql:
         <para>
          Build with <productname>LZ4</productname> compression support.
          This allows the use of <productname>LZ4</productname> for
-         compression of table data.
+         compression of table and WAL data.
         </para>
        </listitem>
       </varlistentry>
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 4f81f19c49..a8794a941a 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -41,10 +41,17 @@
 #define ZLIB_MAX_BLCKSZ		0
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#define	LZ4_MAX_BLCKSZ		LZ4_COMPRESSBOUND(BLCKSZ)
+#else
+#define LZ4_MAX_BLCKSZ		0
+#endif
+
 /* Buffer size required to store a compressed version of backup block image */
 #define PGLZ_MAX_BLCKSZ		PGLZ_MAX_OUTPUT(BLCKSZ)
 
-#define COMPRESS_BUFSIZE	Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ)
+#define COMPRESS_BUFSIZE	Max(Max(PGLZ_MAX_BLCKSZ, ZLIB_MAX_BLCKSZ), LZ4_MAX_BLCKSZ)
 
 /*
  * For each block reference registered with XLogRegisterBuffer, we fill in
@@ -889,6 +896,14 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length,
 		}
 #endif
 
+#ifdef USE_LZ4
+	case WAL_COMPRESSION_LZ4:
+		len = LZ4_compress_fast(source, dest, orig_len, COMPRESS_BUFSIZE, 1);
+		if (len == 0)
+			len = -1;
+		break;
+#endif
+
 	default:
 		/*
 		 * It should be impossible to get here for unsupported algorithms,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 822b2612cd..753b38e58a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -38,6 +38,10 @@
 #include <zlib.h>
 #endif
 
+#ifdef USE_LZ4
+#include "lz4.h"
+#endif
+
 static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
 			pg_attribute_printf(2, 3);
 static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
@@ -72,6 +76,9 @@ const struct config_enum_entry wal_compression_options[] = {
 	{"1", WAL_COMPRESSION_PGLZ, true},
 #ifdef  HAVE_LIBZ
 	{"zlib", WAL_COMPRESSION_ZLIB, false},
+#endif
+#ifdef  USE_LZ4
+	{"lz4", WAL_COMPRESSION_LZ4, false},
 #endif
 	{NULL, 0, false}
 };
@@ -1622,6 +1629,13 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page)
 		}
 #endif
 
+#ifdef USE_LZ4
+		case WAL_COMPRESSION_LZ4:
+			decomp_result = LZ4_decompress_safe(ptr, tmp.data,
+					bkpb->bimg_len, BLCKSZ-bkpb->hole_length);
+			break;
+#endif
+
 		default:
 			report_invalid_record(record, "image at %X/%X is compressed with unsupported codec, block %d (%d/%s)",
 								  (uint32) (record->ReadRecPtr >> 32),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 3991d35afd..dbf3155502 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -218,7 +218,7 @@
 #full_page_writes = on			# recover from partial page writes
 #wal_log_hints = off			# also do full page writes of non-critical updates
 					# (change requires restart)
-#wal_compression = off			# enable compression of full-page writes: off, pglz, zlib
+#wal_compression = off			# enable compression of full-page writes: off, pglz, zlib, lz4
 #wal_init_zero = on			# zero-fill new WAL files
 #wal_recycle = on			# recycle WAL files
 #wal_buffers = -1			# min 32kB, -1 sets based on shared_buffers
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 8b740af66d..0287592cd4 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -330,6 +330,7 @@ typedef enum WalCompression
 	WAL_COMPRESSION_NONE,
 	WAL_COMPRESSION_PGLZ,
 	WAL_COMPRESSION_ZLIB,
+	WAL_COMPRESSION_LZ4,
 } WalCompression;
 
 extern const char *wal_compression_name(WalCompression compression);
diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h
index 2a60c0fb92..abb42b364d 100644
--- a/src/include/access/xlogrecord.h
+++ b/src/include/access/xlogrecord.h
@@ -147,7 +147,7 @@ typedef struct XLogRecordBlockImageHeader
 #define	BKPIMAGE_APPLY		0x02	/* page image should be restored during
 									 * replay */
 #define BKPIMAGE_COMPRESS_METHOD1	0x04	/* bits to encode compression method */
-#define BKPIMAGE_COMPRESS_METHOD2	0x08	/* 0=none, 1=pglz, 2=zlib */
+#define BKPIMAGE_COMPRESS_METHOD2	0x08	/* 0=none, 1=pglz, 2=zlib, 3=lz4 */
 
 /* How many bits to shift to extract compression */
 #define	BKPIMAGE_COMPRESS_OFFSET_BITS	2
-- 
2.17.0

--P+33d92oIH25kiaB
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="v8-0006-Default-to-LZ4.patch"