From 327e86d52be1df8de9c3a324cb06b85ba5db9604 Mon Sep 17 00:00:00 2001
From: David Christensen <david@pgguru.net>
Date: Fri, 29 Sep 2023 15:16:00 -0400
Subject: [PATCH v3 5/5] Add encrypted/authenticated WAL

When using an encrypted cluster, we need to ensure that the WAL is also
encrypted. While we could go with an page-based approach, we use instead a
per-record approach, using GCM for the encryption method and storing the AuthTag
in the xl_crc field.

We change the xl_crc field to instead be a union struct, with a compile-time
adjustable size to allow us to customize the number of bytes allocated to the
GCM authtag.  This allows us to easily adjust the size of bytes needed to
support our authentication.  (Testing has included up to 12, but leaving at this
point to 4 due to keeping the size of the WAL records the same.)
---
 src/backend/access/transam/xlog.c       |  82 ++++--
 src/backend/access/transam/xloginsert.c |  30 ++-
 src/backend/access/transam/xlogreader.c |  17 +-
 src/backend/crypto/bufenc.c             | 344 +++++++++++++++++++++++-
 src/bin/pg_resetwal/pg_resetwal.c       |   6 +-
 src/bin/pg_rewind/meson.build           |   1 +
 src/bin/pg_rewind/pg_rewind.c           |   1 +
 src/bin/pg_waldump/meson.build          |   1 +
 src/bin/pg_waldump/pg_waldump.c         |   1 +
 src/common/cipher_openssl.c             |  53 +++-
 src/include/access/xlog.h               |   8 +
 src/include/access/xlogrecord.h         |  15 +-
 src/include/common/cipher.h             |   7 +
 src/include/crypto/bufenc.h             |   8 +-
 src/test/recovery/t/039_end_of_wal.pl   |   4 +-
 15 files changed, 530 insertions(+), 48 deletions(-)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 332885de95..b8df929009 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -696,7 +696,7 @@ static int	get_sync_bit(int method);
 static void CopyXLogRecordToWAL(int write_len, bool isLogSwitch,
 								XLogRecData *rdata,
 								XLogRecPtr StartPos, XLogRecPtr EndPos,
-								TimeLineID tli);
+								TimeLineID tli, bool encrypt);
 static void ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos,
 									  XLogRecPtr *EndPos, XLogRecPtr *PrevPtr);
 static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos,
@@ -732,7 +732,7 @@ static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
  *
  * The first XLogRecData in the chain must be for the record header, and its
  * data must be MAXALIGNed.  XLogInsertRecord fills in the xl_prev and
- * xl_crc fields in the header, the rest of the header must already be filled
+ * xl_integrity fields in the header, the rest of the header must already be filled
  * by the caller.
  *
  * Returns XLOG pointer to end of record (beginning of next record).
@@ -905,20 +905,28 @@ XLogInsertRecord(XLogRecData *rdata,
 	{
 		/*
 		 * Now that xl_prev has been filled in, calculate CRC of the record
-		 * header.
+		 * header.  If we are using encrypted WAL, this CRC is overwritten by
+		 * the authentication tag, so just zero
 		 */
-		rdata_crc = rechdr->xl_crc;
-		COMP_CRC32C(rdata_crc, rechdr, offsetof(XLogRecord, xl_crc));
-		FIN_CRC32C(rdata_crc);
-		rechdr->xl_crc = rdata_crc;
+		if (!encrypt_wal)
+		{
+			rdata_crc = rechdr->xl_integrity.crc;
+			COMP_CRC32C(rdata_crc, rechdr, offsetof(XLogRecord, xl_integrity.crc));
+			FIN_CRC32C(rdata_crc);
+			rechdr->xl_integrity.crc = rdata_crc;
+		}
+		else
+			memset(&rechdr->xl_integrity, 0, sizeof(rechdr->xl_integrity));
 
 		/*
 		 * All the record data, including the header, is now ready to be
-		 * inserted. Copy the record in the space reserved.
+		 * inserted. Copy the record in the space reserved.  If WAL is
+		 * encrypted, we will also calculate the authentication tag and
+		 * perform the encryption of data as it is being written out.
 		 */
 		CopyXLogRecordToWAL(rechdr->xl_tot_len,
 							class == WALINSERT_SPECIAL_SWITCH, rdata,
-							StartPos, EndPos, insertTLI);
+							StartPos, EndPos, insertTLI, encrypt_wal);
 
 		/*
 		 * Unless record is flagged as not important, update LSN of last
@@ -1215,17 +1223,30 @@ ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecPtr *PrevPtr)
 
 /*
  * Subroutine of XLogInsertRecord.  Copies a WAL record to an already-reserved
- * area in the WAL.
+ * area in the WAL.  If "encrypt" is true, encrypt the non-header portions of
+ * the record, storing the GCM Tag in the initial header's xl_integrity field.
  */
 static void
 CopyXLogRecordToWAL(int write_len, bool isLogSwitch, XLogRecData *rdata,
-					XLogRecPtr StartPos, XLogRecPtr EndPos, TimeLineID tli)
+					XLogRecPtr StartPos, XLogRecPtr EndPos, TimeLineID tli,
+					bool encrypt)
 {
 	char	   *currpos;
 	int			freespace;
 	int			written;
 	XLogRecPtr	CurrPos;
 	XLogPageHeader pagehdr;
+	char authtag[XL_AUTHTAG_SIZE] = {0};
+
+	/* If encrypting we will need to precalculate the authtag for this record
+	 * and set xl_integrity to it. */
+	if (encrypt)
+	{
+		CalculateXLogRecordAuthtag(rdata, StartPos, authtag);
+		memcpy(&((XLogRecord*)rdata->data)->xl_integrity, authtag, XL_AUTHTAG_SIZE);
+		/* Start the encryption process; initializes record IV and precalculates AAD */
+		StartEncryptXLogRecord((XLogRecord*)rdata->data, StartPos);
+	}
 
 	/*
 	 * Get a pointer to the right place in the right WAL buffer to start
@@ -1254,7 +1275,10 @@ CopyXLogRecordToWAL(int write_len, bool isLogSwitch, XLogRecData *rdata,
 			 * Write what fits on this page, and continue on the next page.
 			 */
 			Assert(CurrPos % XLOG_BLCKSZ >= SizeOfXLogShortPHD || freespace == 0);
-			memcpy(currpos, rdata_data, freespace);
+			if (encrypt)
+				EncryptXLogRecordIncremental(rdata_data, currpos, freespace);
+			else
+				memcpy(currpos, rdata_data, freespace);
 			rdata_data += freespace;
 			rdata_len -= freespace;
 			written += freespace;
@@ -1289,7 +1313,10 @@ CopyXLogRecordToWAL(int write_len, bool isLogSwitch, XLogRecData *rdata,
 		}
 
 		Assert(CurrPos % XLOG_BLCKSZ >= SizeOfXLogShortPHD || rdata_len == 0);
-		memcpy(currpos, rdata_data, rdata_len);
+		if (encrypt)
+			EncryptXLogRecordIncremental(rdata_data, currpos, rdata_len);
+		else
+			memcpy(currpos, rdata_data, rdata_len);
 		currpos += rdata_len;
 		CurrPos += rdata_len;
 		freespace -= rdata_len;
@@ -1299,6 +1326,9 @@ CopyXLogRecordToWAL(int write_len, bool isLogSwitch, XLogRecData *rdata,
 	}
 	Assert(written == write_len);
 
+	if (encrypt)
+		FinishEncryptXLogRecord(currpos);
+
 	/*
 	 * If this was an xlog-switch, it's not enough to write the switch record,
 	 * we also have to consume all the remaining space in the WAL segment.  We
@@ -4848,11 +4878,27 @@ BootStrapXLOG(void)
 	BootStrapKmgr();
 	InitializeBufferEncryption(bootstrap_file_encryption_method);
 
-	INIT_CRC32C(crc);
-	COMP_CRC32C(crc, ((char *) record) + SizeOfXLogRecord, record->xl_tot_len - SizeOfXLogRecord);
-	COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc));
-	FIN_CRC32C(crc);
-	record->xl_crc = crc;
+	if (bootstrap_file_encryption_method != DISABLED_ENCRYPTION_METHOD)
+	{
+		/* Since this must be set for XLogBytePosToRecPtr to work properly,
+		 * set explicitly here.  Note that we don't actually care about this
+		 * exact calculation being correct with non-default wal_segment_size
+		 * since it will be reset when we hit ReadControlFile() later in this
+		 * function, and as the first record in the WAL stream this will
+		 * always be the first position. */
+		UsableBytesInSegment =
+			(wal_segment_size / XLOG_BLCKSZ * UsableBytesInPage) -
+			(SizeOfXLogLongPHD - SizeOfXLogShortPHD);
+		EncryptXLogRecord(record, wal_segment_size + SizeOfXLogLongPHD, NULL);
+	}
+	else
+	{
+		INIT_CRC32C(crc);
+		COMP_CRC32C(crc, ((char *) record) + SizeOfXLogRecord, record->xl_tot_len - SizeOfXLogRecord);
+		COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_integrity));
+		FIN_CRC32C(crc);
+		record->xl_integrity.crc = crc;
+	}
 
 	/* Create first XLOG segment file */
 	openLogTLI = BootstrapTimeLineID;
diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c
index 7c2277dd5b..24441a8590 100644
--- a/src/backend/access/transam/xloginsert.c
+++ b/src/backend/access/transam/xloginsert.c
@@ -893,18 +893,21 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
 	hdr_rdt.len = (scratch - hdr_scratch);
 	total_len += hdr_rdt.len;
 
-	/*
-	 * Calculate CRC of the data
-	 *
-	 * Note that the record header isn't added into the CRC initially since we
-	 * don't know the prev-link yet.  Thus, the CRC will represent the CRC of
-	 * the whole record in the order: rdata, then backup blocks, then record
-	 * header.
-	 */
-	INIT_CRC32C(rdata_crc);
-	COMP_CRC32C(rdata_crc, hdr_scratch + SizeOfXLogRecord, hdr_rdt.len - SizeOfXLogRecord);
-	for (rdt = hdr_rdt.next; rdt != NULL; rdt = rdt->next)
-		COMP_CRC32C(rdata_crc, rdt->data, rdt->len);
+	if (!encrypt_wal)
+	{
+		/*
+		 * Calculate CRC of the data
+		 *
+		 * Note that the record header isn't added into the CRC initially since we
+		 * don't know the prev-link yet.  Thus, the CRC will represent the CRC of
+		 * the whole record in the order: rdata, then backup blocks, then record
+		 * header.
+		 */
+		INIT_CRC32C(rdata_crc);
+		COMP_CRC32C(rdata_crc, hdr_scratch + SizeOfXLogRecord, hdr_rdt.len - SizeOfXLogRecord);
+		for (rdt = hdr_rdt.next; rdt != NULL; rdt = rdt->next)
+			COMP_CRC32C(rdata_crc, rdt->data, rdt->len);
+	}
 
 	/*
 	 * Ensure that the XLogRecord is not too large.
@@ -929,7 +932,8 @@ XLogRecordAssemble(RmgrId rmid, uint8 info,
 	rechdr->xl_info = info;
 	rechdr->xl_rmid = rmid;
 	rechdr->xl_prev = InvalidXLogRecPtr;
-	rechdr->xl_crc = rdata_crc;
+	if (!encrypt_wal)
+		rechdr->xl_integrity.crc = rdata_crc;
 
 	return &hdr_rdt;
 }
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index e0baa86bd3..f006ea03c5 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -31,6 +31,7 @@
 #include "access/xlogrecord.h"
 #include "catalog/pg_control.h"
 #include "common/pg_lzcompress.h"
+#include "crypto/bufenc.h"
 #include "replication/origin.h"
 
 #ifndef FRONTEND
@@ -806,6 +807,11 @@ restart:
 		Assert(gotheader);
 
 		record = (XLogRecord *) state->readRecordBuf;
+
+		/* here we decrypt the record as needed */
+		if (encrypt_wal && !DecryptXLogRecord(record, RecPtr))
+			goto err;
+
 		if (!ValidXLogRecord(state, record, RecPtr))
 			goto err;
 
@@ -825,6 +831,9 @@ restart:
 			goto err;
 
 		/* Record does not cross a page boundary */
+		if (encrypt_wal && !DecryptXLogRecord(record, RecPtr))
+			goto err;
+
 		if (!ValidXLogRecord(state, record, RecPtr))
 			goto err;
 
@@ -1169,15 +1178,19 @@ ValidXLogRecord(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr)
 	pg_crc32c	crc;
 
 	Assert(record->xl_tot_len >= SizeOfXLogRecord);
+	/* encrypted WAL is guaranteed to be valid since we are using authenticated encryption */
+
+	if (encrypt_wal)
+		return true;
 
 	/* Calculate the CRC */
 	INIT_CRC32C(crc);
 	COMP_CRC32C(crc, ((char *) record) + SizeOfXLogRecord, record->xl_tot_len - SizeOfXLogRecord);
 	/* include the record header last */
-	COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc));
+	COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_integrity.crc));
 	FIN_CRC32C(crc);
 
-	if (!EQ_CRC32C(record->xl_crc, crc))
+	if (!EQ_CRC32C(record->xl_integrity.crc, crc))
 	{
 		report_invalid_record(state,
 							  "incorrect resource manager data checksum in record at %X/%X",
diff --git a/src/backend/crypto/bufenc.c b/src/backend/crypto/bufenc.c
index 18a34d9802..33b4cfd9a7 100644
--- a/src/backend/crypto/bufenc.c
+++ b/src/backend/crypto/bufenc.c
@@ -18,12 +18,11 @@
 
 #include "access/gist.h"
 #include "access/xlog.h"
+#include "access/xlog_internal.h"
 #include "crypto/bufenc.h"
 #include "storage/bufpage.h"
 #include "storage/fd.h"
 
-extern XLogRecPtr LSNForEncryption(bool use_wal_lsn);
-
 /*
  * We use the page LSN, page number, and permanent-bit to indicate if a fake
  * LSN was used to create a nonce for each page.
@@ -31,6 +30,7 @@ extern XLogRecPtr LSNForEncryption(bool use_wal_lsn);
 #define BUFENC_IV_SIZE		16
 
 static unsigned char buf_encryption_iv[BUFENC_IV_SIZE];
+static unsigned char xlog_encryption_iv[BUFENC_IV_SIZE] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
 static int file_encryption_tag_size = 0;
 static int file_encryption_page_size = 0;
 static int file_encryption_method = DISABLED_ENCRYPTION_METHOD;
@@ -49,6 +49,10 @@ AdditionalAuthenticatedData auth_data;
 
 PgCipherCtx *BufEncCtx = NULL;
 PgCipherCtx *BufDecCtx = NULL;
+PgCipherCtx *XLogEncCtx = NULL;
+PgCipherCtx *XLogDecCtx = NULL;
+
+EncryptionHandle encr_state;
 
 static void set_buffer_encryption_iv(Page page, BlockNumber blkno,
 									 bool relation_is_permanent);
@@ -88,8 +92,25 @@ InitializeBufferEncryption(int init_file_encryption_method)
 
 	file_encryption_tag_size = SizeOfEncryptionTag(file_encryption_method);
 	file_encryption_page_size = SizeOfPageEncryption(file_encryption_method);
+
+	key = KmgrGetKey(KMGR_KEY_ID_WAL);
+
+	XLogEncCtx = pg_cipher_ctx_create(PG_CIPHER_AES_GCM,
+									  (unsigned char *) key->key,
+									  EncryptionBlockLength(file_encryption_method),
+									  true);
+	if (!XLogEncCtx)
+		my_error("cannot initialize xlog encryption context");
+
+	XLogDecCtx = pg_cipher_ctx_create(PG_CIPHER_AES_GCM,
+									  (unsigned char *) key->key,
+									  EncryptionBlockLength(file_encryption_method),
+									  false);
+	if (!XLogDecCtx)
+		my_error("cannot initialize xlog decryption context");
 }
 
+
 /* Encrypt the given page with the relation key */
 void
 EncryptPage(Page page, bool relation_is_permanent, BlockNumber blkno, RelFileNumber fileno)
@@ -235,3 +256,322 @@ setup_additional_authenticated_data(Page page, BlockNumber blkno,
 	auth_data.fileno = fileno;
 	auth_data.blkNo = blkno;
 }
+
+
+/* TODO:
+ *
+ * - move XLog pieces to the actual xlog source, but expose these encryption
+ *   contexts so we can use them there
+ *
+ * - abstract the incremental Start/Iterate/Finish routines where we are using
+ *   openssl directly and move to the crypto/cipher*.c files.
+ */
+
+/*
+ * Encrypt an initialized XLogRecord with the xlog key, storing auth in the
+ * xl_integrity field.  Anything before xl_integrity is AAD and anything after
+ * is encrypted.
+ *
+ * The *dest field is assumed to be preallocated, reserved space (likely
+ * already in the wal_buffers) where we should copy the final assembled
+ * records.  It should be record->xl_tot_len bytes in space, and the caller
+ * needs to reserve this space ahead of time and take all appropriate locks.
+ */
+
+// TODO: this needs to be fixed to work with other-sized authtags and additional AAD checks/changes
+void
+EncryptXLogRecord(XLogRecord *record, XLogRecPtr address, char *dest)
+{
+	unsigned char *ptr = (unsigned char*)record + SizeOfXLogRecord;
+	int enclen = 0;
+
+	/* sanity check to ensure we are not encrypting an already encrypted record */
+	Assert(*(uint64*)record->xl_integrity.authtag== 0);
+
+	/* setup IV based on the xlp_pageaddr field */
+	memcpy(xlog_encryption_iv, &address, sizeof(address));
+
+	if (unlikely(!pg_cipher_encrypt(XLogEncCtx, PG_CIPHER_AES_GCM,
+									ptr,	/* input */
+									record->xl_tot_len - SizeOfXLogRecord,
+									ptr,	/* output */
+									&enclen,	/* resulting length */
+									xlog_encryption_iv,	/* iv */
+									BUFENC_IV_SIZE,
+									(unsigned char*)record, offsetof(XLogRecord, xl_integrity), /* AAD */
+									record->xl_integrity.authtag, sizeof(record->xl_integrity.authtag))))
+	{
+		my_error("cannot encrypt xlog page %lu", address);
+		return;
+	}
+
+	Assert(enclen + SizeOfXLogRecord == record->xl_tot_len);
+}
+
+/*
+ * Decrypt an encrypted XLogRecord with the xlog key, validating against auth
+ * in the xl_integrity field.  Anything before xl_integrity is AAD and
+ * anything after is encrypted.
+ */
+bool
+DecryptXLogRecord(XLogRecord *record, XLogRecPtr address)
+{
+	unsigned char *ptr = (unsigned char*)record + SizeOfXLogRecord;
+	int declen = 0;
+
+	/* early abort if we are already decrypted */
+	if (!*(uint64*)record->xl_integrity.authtag)
+		return true;
+
+	/* setup IV based on the xlp_pageaddr field */
+	memcpy(xlog_encryption_iv, &address, sizeof(address));
+
+	if (unlikely(!pg_cipher_decrypt(XLogDecCtx, PG_CIPHER_AES_GCM,
+									ptr,	/* input */
+									record->xl_tot_len - SizeOfXLogRecord,
+									ptr,	/* output */
+									&declen,	/* resulting length */
+									xlog_encryption_iv,	/* iv */
+									BUFENC_IV_SIZE,
+									(unsigned char*)record, offsetof(XLogRecord, xl_integrity), /* AAD */
+									/* NULL, 0))) */
+									(unsigned char*)&record->xl_integrity, sizeof(record->xl_integrity))))
+		return false;
+
+	/* in-memory decoded records have xl_integrity of 0 when encryption is defined */
+	memset(record->xl_integrity.authtag, 0, sizeof(record->xl_integrity.authtag));
+
+	return (declen + SizeOfXLogRecord == record->xl_tot_len);
+}
+
+/*
+ * Calculate the GCM Authtag for the given XLogRecord and store the 64-bit
+ * value in the given address.
+ *
+ * This calculates the tag in the same way as would be done with the standard
+ * encryption, essentially authenticating the XLogRecord header (minus the
+ * xl_integrity field) and then processing the rest of the record after this
+ * field as one contiguous block.
+ *
+ * The reason we need a separate routine for this is because the XLogRecord as
+ * given here may be in a buffer we cannot encrypt in-place, say if the same
+ * memory location were used for the unencrypted streaming replication
+ * XLogRecord.  We will still need to update the xl_integrity field in this
+ * case.
+ *
+ * XXX: if we are using the xl_integrity field for CRC replacement, do we in
+ * actuality /need/ to leave encrypted so we can have a single code path on
+ * streaming rep for decryption?  Otherwise how would we know the end of valid
+ * records?  Look into this more.
+ */
+void
+CalculateXLogRecordAuthtag(XLogRecData *recdata, XLogRecPtr address, char *tag)
+{
+	XLogRecData *rdt;
+	XLogRecord *recheader;
+	int len;
+#define SCRATCH_SIZE 1024
+	unsigned char scratch[SCRATCH_SIZE];
+	char authtag[XL_AUTHTAG_SIZE];
+
+	/*
+	 * Unfortunately, in order to get the right value for the authtag, it is
+	 * not sufficient to just EVP_EncryptUpdate() over a NULL buffer, as we do
+	 * for AAD, so we need to utilize a "scribble buffer" in order to store
+	 * temporarily encrypted results though we don't end up doing anything
+	 * with them.
+	 *
+	 * This does mean that currently we have to effectively encrypt the
+	 * XLogRecord twice -- one to pre-calculate the authtag which needs to be
+	 * stored in the initial XLogRecData in order to handle these things
+	 * incrementally, and once to encrypt as we copy data into
+	 * CopyXLogRecordToWAL().  Unfortunately, this is unavoidable as the
+	 * contract of GetWALBuffer() indicates that we cannot reference any
+	 * earlier buffers, as these may end up flushing the first buffer we need
+	 * to store the actual hash into.
+	 */
+
+	/* verify we're a sensible chain of data */
+	Assert(recdata != NULL && recdata->data != NULL);
+	recheader = (XLogRecord*)(recdata->data);
+
+	/* make sure our first block looks like a normal XLogRecord header */
+	Assert(recdata->len >= SizeOfXLogRecord);
+
+	/* initialize our context */
+	/* setup IV based on the xlp_pageaddr field */
+	memcpy(xlog_encryption_iv, &address, sizeof(address));
+
+	/* initialize our IV context */
+
+	encr_state = pg_cipher_incr_init(XLogEncCtx, PG_CIPHER_AES_GCM,
+									 xlog_encryption_iv, 16);
+
+	/* initial AAD is not full length, so handle this page separately */
+	if (!pg_cipher_incr_add_authenticated_data(
+			encr_state,
+			(unsigned char*)recheader,
+			offsetof(XLogRecord,xl_integrity)))
+		my_error("error when trying to update AAD");
+
+	/* also for initial page, anything past SizeOfXLogRecord needs to be
+	 * added */
+	if (recdata->len > SizeOfXLogRecord)
+	{
+		int mylen = recdata->len - SizeOfXLogRecord;
+		unsigned char *ptr = (unsigned char *)recdata->data + SizeOfXLogRecord;
+		int step;
+
+		/* since we have to write to a scribble buffer to get the right answer
+		 * (boo), break into chunks of SCRATCH_SIZE */
+		do {
+			step = mylen > SCRATCH_SIZE ? SCRATCH_SIZE : mylen;
+
+			if (!pg_cipher_incr_encrypt(encr_state, ptr, step, scratch, &len))
+				my_error("error when trying to update data");
+			ptr += step;
+		} while (step > 0 && (mylen -= step) > 0);
+	}
+
+	/* progressively update with the data pages for the record, chunking into
+	 * SCRATCH_SIZE chunks */
+	for (rdt = recdata->next; rdt != NULL; rdt = rdt->next)
+	{
+		int mylen = rdt->len;
+		unsigned char *ptr = (unsigned char*)rdt->data;
+		int step;
+
+		do {
+			step = mylen > SCRATCH_SIZE ? SCRATCH_SIZE : mylen;
+
+			if (!pg_cipher_incr_encrypt(encr_state, ptr, step, scratch, &len))
+				my_error("error when trying to update data");
+			ptr += step;
+		} while (step > 0 && (mylen -= step) > 0);
+	}
+
+    /*
+	 * Finalize the encryption, which could add more to output, and extract
+	 * our authtag.
+	 */
+	pg_cipher_incr_finish(encr_state, scratch, &len, (unsigned char*)authtag, XL_AUTHTAG_SIZE);
+	memcpy(tag,authtag,XL_AUTHTAG_SIZE);
+}
+
+/* Incremental XLog Record Encryption */
+
+/*
+ * This routine initializes the machinery for the initial state for encrypting
+ * an XLogRecord while moving into the reserved WAL space.  It sets some
+ * global variables, assuming nothing else is intervening in subsequent calls
+ * here, with the following sequence of events happening:
+ *
+ * 1. StartEncryptXLogRecord() to initialize state
+ * 2. EncryptXLogRecordIncremental() to copy or encrypt some number of bytes
+ *    (depending on where in the record we are)
+ * 3. FinishEncryptXLogRecord() to
+ *    finalize state once all data has been copied or encrypted
+ *
+ * Since only one XLogRecord can be inserted at a time for a single backend,
+ * we do not need to worry about overwriting state here, so use globals for state mgmt
+ */
+
+static int bytes_processed;
+static int bytes_tot;
+static char xrechdr[SizeOfXLogRecord];
+
+void StartEncryptXLogRecord(XLogRecord *record, XLogRecPtr address)
+{
+	/* setup IV based on the xlp_pageaddr field */
+	memcpy(xlog_encryption_iv, &address, sizeof(address));
+
+	/* initialize our IV context */
+	encr_state = pg_cipher_incr_init(XLogEncCtx, PG_CIPHER_AES_GCM,
+									 xlog_encryption_iv, 16);
+
+	if (!encr_state)
+		my_error("Couldn't initialize incremental encryption context");
+
+	/* reset our other state vars */
+	bytes_processed = 0;
+	bytes_tot = record->xl_tot_len;
+}
+
+/* returns the byte offset copied to the output buffer; can be <= input length if we are still filling the header */
+int EncryptXLogRecordIncremental(char *plaintext, char *encdest, int len)
+{
+	/* ensure we're not trying to process too much data */
+	Assert(bytes_processed + len <= bytes_tot);
+
+	/* are we just copying data? */
+	if (bytes_processed < SizeOfXLogRecord)
+	{
+		int remaining = SizeOfXLogRecord - bytes_processed;
+
+		if (len >= remaining)
+		{
+			/* copy remaining bytes into our local XLogRecord header buffer */
+			memcpy(xrechdr + bytes_processed, plaintext, remaining);
+
+			/* copy remaining bytes into output stream */
+			memcpy(encdest, plaintext, remaining);
+
+			/* adjust counts for what we've done for later encrypted processing in this round */
+			bytes_processed += remaining;
+			encdest += remaining;
+			plaintext += remaining;
+			len -= remaining;
+
+			/* initialize our encryption for the record minus the xl_integrity
+			 * field, which is calculated in an initial pass without
+			 * encrypting the underlying data. */
+			pg_cipher_incr_add_authenticated_data(
+				encr_state,
+				(unsigned char*)xrechdr,
+				offsetof(XLogRecord, xl_integrity)
+			);
+
+			/* at this point, we're done with the unencrypted header and any
+			 * further data will be encrypted incrementally */
+		}
+		else
+		{
+			/* only partial bytes available, so let's just copy into our buffer and output buffer */
+			memcpy(xrechdr + bytes_processed, plaintext, len);
+			memcpy(encdest, plaintext, len);
+			bytes_processed += len;
+			len = 0;
+		}
+	}
+
+	/* incrementally encrypt data */
+	if (len > 0)
+	{
+		int enclen;
+
+		pg_cipher_incr_encrypt(
+			encr_state,
+			(unsigned char*)plaintext,
+			len,
+			(unsigned char*)encdest,
+			&enclen
+		);
+		bytes_processed += enclen;
+	}
+	return bytes_processed;
+}
+
+void FinishEncryptXLogRecord(char *loc)
+{
+	int len;
+	unsigned char tag[XL_AUTHTAG_SIZE] = {0};
+
+	Assert(bytes_processed <= bytes_tot);
+
+    /* Finalize the encryption, which could add more to output. */
+	pg_cipher_incr_finish(encr_state, (unsigned char*)loc, &len, tag, XL_AUTHTAG_SIZE);
+	bytes_processed += len;
+
+	/* ensure we copied all the data we expected */
+	Assert(bytes_processed == bytes_tot);
+}
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index ef6d50c93b..8fae962a0c 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -1083,11 +1083,13 @@ WriteEmptyXLOG(void)
 	memcpy(recptr, &ControlFile.checkPointCopy,
 		   sizeof(CheckPoint));
 
+	/* XXX - TODO: handle w/encrypted WAL too */
+	
 	INIT_CRC32C(crc);
 	COMP_CRC32C(crc, ((char *) record) + SizeOfXLogRecord, record->xl_tot_len - SizeOfXLogRecord);
-	COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc));
+	COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_integrity));
 	FIN_CRC32C(crc);
-	record->xl_crc = crc;
+	record->xl_integrity.crc = crc;
 
 	/* Write the first page */
 	XLogFilePath(path, ControlFile.checkPointCopy.ThisTimeLineID,
diff --git a/src/bin/pg_rewind/meson.build b/src/bin/pg_rewind/meson.build
index fd22818be4..f1e3b51818 100644
--- a/src/bin/pg_rewind/meson.build
+++ b/src/bin/pg_rewind/meson.build
@@ -12,6 +12,7 @@ pg_rewind_sources = files(
 )
 
 pg_rewind_sources += xlogreader_sources
+pg_rewind_sources += files('../../backend/crypto/bufenc.c')
 
 if host_system == 'windows'
   pg_rewind_sources += rc_bin_gen.process(win32ver_rc, extra_args: [
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index bfd44a284e..7769c78f51 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -62,6 +62,7 @@ static ControlFileData ControlFile_source_after;
 
 const char *progname;
 int			WalSegSz;
+bool encrypt_wal = false;
 
 /* Configuration options */
 char	   *datadir_target = NULL;
diff --git a/src/bin/pg_waldump/meson.build b/src/bin/pg_waldump/meson.build
index ae674d17c3..c96b797d6c 100644
--- a/src/bin/pg_waldump/meson.build
+++ b/src/bin/pg_waldump/meson.build
@@ -9,6 +9,7 @@ pg_waldump_sources = files(
 pg_waldump_sources += rmgr_desc_sources
 pg_waldump_sources += xlogreader_sources
 pg_waldump_sources += files('../../backend/access/transam/xlogstats.c')
+pg_waldump_sources += files('../../backend/crypto/bufenc.c')
 
 if host_system == 'windows'
   pg_waldump_sources += rc_bin_gen.process(win32ver_rc, extra_args: [
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index a3535bdfa9..49c955319e 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -80,6 +80,7 @@ typedef struct XLogDumpConfig
 	char	   *save_fullpage_path;
 } XLogDumpConfig;
 
+bool encrypt_wal = false;
 
 /*
  * When sigint is called, just tell the system to exit at the next possible
diff --git a/src/common/cipher_openssl.c b/src/common/cipher_openssl.c
index d1d4581912..a348799b70 100644
--- a/src/common/cipher_openssl.c
+++ b/src/common/cipher_openssl.c
@@ -96,8 +96,45 @@ pg_cipher_encrypt(PgCipherCtx *ctx, int cipher,
 				  const unsigned char *aad, const int aadlen,
 				  unsigned char *outtag, const int taglen)
 {
-	int			len;
-	int			enclen;
+	return pg_cipher_encrypt_ex(ctx, cipher, &plaintext, &inlen, 1,
+							 ciphertext,outlen,iv,ivlen,aad,aadlen,outtag,taglen);
+}
+
+/*
+ * Encryption routine to encrypt multiple data blocks of source data.
+ *
+ * ctx is the encryption context which must have been created previously.
+ *
+ * plaintext is an array of pointers to data we are going to encrypt
+ * inlen is an array the length of the data to encrypt
+ * nchunks are the number of elements in each of these arrays, which must match
+ *
+ * ciphertext is the encrypted result
+ * outlen is the encrypted length
+ *
+ * iv is the IV to use.
+ * ivlen is the IV length to use.
+ *
+ * aad is a pointer to additional authenticated data.
+ * aadlen is the length of aad.
+ *
+ * aad is a pointer to additional authenticated data.
+ * aadlen is the length of aad.
+ *
+ * outtag is the resulting tag.
+ * taglen is the length of the tag.
+ */
+bool
+pg_cipher_encrypt_ex(PgCipherCtx *ctx, int cipher,
+				  const unsigned char **plaintext, const int *inlen,
+				  const int nchunks,
+				  unsigned char *ciphertext, int *outlen,
+				  const unsigned char *iv, const int ivlen,
+				  const unsigned char *aad, const int aadlen,
+				  unsigned char *outtag, const int taglen)
+{
+	int			len = 0;
+	int			enclen = 0;
 
 	Assert(ctx != NULL);
 
@@ -123,12 +160,20 @@ pg_cipher_encrypt(PgCipherCtx *ctx, int cipher,
 	if (aad && aadlen && !EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen))
 		return false;
 
+	/* reset to zero since we only care about the total /encrypted/ length */
+	len = 0;
+
 	/*
 	 * This is the function which is actually performing the encryption for
 	 * us.
 	 */
-	if (!EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, inlen))
-		return false;
+	for (int i = 0; i < nchunks; i++) {
+		int inclen;
+
+		if (!EVP_EncryptUpdate(ctx, ciphertext + len, &inclen, plaintext[i], inlen[i]))
+			return false;
+		len += inclen;
+	}
 
 	enclen = len;
 
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 3eb8ce2a17..d910f81753 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -93,6 +93,7 @@ typedef enum RecoveryState
 } RecoveryState;
 
 extern PGDLLIMPORT int wal_level;
+extern bool encrypt_wal;
 
 /* Is WAL archiving enabled (always or only while server is running normally)? */
 #define XLogArchivingActive() \
@@ -130,6 +131,13 @@ extern PGDLLIMPORT int wal_level;
 extern PGDLLIMPORT bool XLOG_DEBUG;
 #endif
 
+#ifdef FRONTEND
+/* TODO: handle in FRONTEND */
+#define encrypt_wal 0
+#else
+#define encrypt_wal (GetFileEncryptionMethod())
+#endif
+
 /*
  * OR-able request flag bits for checkpoints.  The "cause" bits are used only
  * for logging purposes.  Note: the flags must be defined so that it's
diff --git a/src/include/access/xlogrecord.h b/src/include/access/xlogrecord.h
index ec9a3c802a..31edb7e487 100644
--- a/src/include/access/xlogrecord.h
+++ b/src/include/access/xlogrecord.h
@@ -38,6 +38,10 @@
  * XLogRecordDataHeaderLong structs all begin with a single 'id' byte. It's
  * used to distinguish between block references, and the main data structs.
  */
+
+#define XL_AUTHTAG_SIZE 4
+#define XL_HEADER_PAD 2
+
 typedef struct XLogRecord
 {
 	uint32		xl_tot_len;		/* total len of entire record */
@@ -45,14 +49,16 @@ typedef struct XLogRecord
 	XLogRecPtr	xl_prev;		/* ptr to previous record in log */
 	uint8		xl_info;		/* flag bits, see below */
 	RmgrId		xl_rmid;		/* resource manager for this record */
-	/* 2 bytes of padding here, initialize to zero */
-	pg_crc32c	xl_crc;			/* CRC for this record */
-
+	uint8       xl_pad[XL_HEADER_PAD];		/* required alignment padding */
+	union {
+		uint32 crc;
+		unsigned char authtag[XL_AUTHTAG_SIZE];			/* CRC or tag for this record */
+	} xl_integrity;
 	/* XLogRecordBlockHeaders and XLogRecordDataHeader follow, no padding */
 
 } XLogRecord;
 
-#define SizeOfXLogRecord	(offsetof(XLogRecord, xl_crc) + sizeof(pg_crc32c))
+#define SizeOfXLogRecord	(offsetof(XLogRecord, xl_integrity) + XL_AUTHTAG_SIZE)
 
 /*
  * The high 4 bits in xl_info may be used freely by rmgr. The
@@ -90,6 +96,7 @@ typedef struct XLogRecord
  */
 #define XLR_CHECK_CONSISTENCY	0x02
 
+
 /*
  * Header info for block data appended to an XLOG record.
  *
diff --git a/src/include/common/cipher.h b/src/include/common/cipher.h
index 41de383857..faca5c1685 100644
--- a/src/include/common/cipher.h
+++ b/src/include/common/cipher.h
@@ -64,6 +64,13 @@ extern bool pg_cipher_encrypt(PgCipherCtx *ctx, int cipher,
 							  const unsigned char *iv, const int ivlen,
 							  const unsigned char *aad, const int aadlen,
 							  unsigned char *tag, const int taglen);
+extern bool pg_cipher_encrypt_ex(PgCipherCtx *ctx, int cipher,
+							  const unsigned char **plaintext, const int *inlen,
+							  int nchunks,
+							  unsigned char *ciphertext, int *outlen,
+							  const unsigned char *iv, const int ivlen,
+							  const unsigned char *aad, const int aadlen,
+							  unsigned char *tag, const int taglen);
 extern bool pg_cipher_decrypt(PgCipherCtx *ctx, const int cipher,
 							  const unsigned char *ciphertext, const int inlen,
 							  unsigned char *plaintext, int *outlen,
diff --git a/src/include/crypto/bufenc.h b/src/include/crypto/bufenc.h
index 41270ee923..0d9a467e40 100644
--- a/src/include/crypto/bufenc.h
+++ b/src/include/crypto/bufenc.h
@@ -13,6 +13,7 @@
 
 #include "storage/bufmgr.h"
 #include "crypto/kmgr.h"
+#include "access/xlog_internal.h"
 
 /* Cluster encryption encrypts only main forks */
 #define PageNeedsToBeEncrypted(forknum) \
@@ -33,5 +34,10 @@ extern void EncryptPage(Page page, bool relation_is_permanent,
 						BlockNumber blkno, RelFileNumber fileno);
 extern void DecryptPage(Page page, bool relation_is_permanent,
 						BlockNumber blkno, RelFileNumber fileno);
-
+extern void EncryptXLogRecord(XLogRecord *record, XLogRecPtr address, char *dest);
+extern bool DecryptXLogRecord(XLogRecord *record, XLogRecPtr address);
+extern void CalculateXLogRecordAuthtag(XLogRecData *recdat, XLogRecPtr address, char *tag);
+extern void StartEncryptXLogRecord(XLogRecord *record, XLogRecPtr address);
+extern int EncryptXLogRecordIncremental(char *plaintext, char *encdest, int len);
+extern void FinishEncryptXLogRecord(char *loc);
 #endif							/* BUFENC_H */
diff --git a/src/test/recovery/t/039_end_of_wal.pl b/src/test/recovery/t/039_end_of_wal.pl
index d2bf062bb2..486b824ec8 100644
--- a/src/test/recovery/t/039_end_of_wal.pl
+++ b/src/test/recovery/t/039_end_of_wal.pl
@@ -137,11 +137,11 @@ sub build_record_header
 	# C for xl_rmid
 	# BB for two bytes of padding
 	# I for xl_crc
-	return pack("IIIICCBBI",
+	return pack("IIIICCBBIII",
 		$xl_tot_len, $xl_xid,
 		$BIG_ENDIAN ? 0        : $xl_prev,
 		$BIG_ENDIAN ? $xl_prev : 0,
-		$xl_info, $xl_rmid, 0, 0, $xl_crc);
+		$xl_info, $xl_rmid, 0, 0, $xl_crc, 0, 0);
 }
 
 # Build a fake WAL page header, based on the data given by the caller
-- 
2.40.1

