($INBOX_DIR/description missing)  

help / color / mirror / Atom feed

[PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
26+ messages / 2 participants
[nested] [flat]

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9a56b3487e..b64c61f742 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1324,7 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index eccb88b611..ec78de5dd9 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1213,7 +1213,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6469,8 +6472,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index d829e57baa..582b1dd11b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -422,7 +425,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -433,7 +436,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -479,10 +482,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -552,7 +555,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -579,7 +582,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -631,7 +634,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -658,7 +661,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -792,10 +795,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -808,7 +811,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -832,7 +835,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -968,7 +971,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1022,10 +1025,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1146,7 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1180,8 +1183,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 97def6f781..ee322ba796 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -189,11 +189,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 33d952f9fd..a0914ddb25 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -578,8 +578,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 7344a7c692..c5881594b7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -112,7 +112,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -172,7 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9329a6a6d6..025479431d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1043,7 +1043,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 732d463a49..e9d7d70db7 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.18.4


----Next_Part(Tue_Sep__8_16_35_16_2020_701)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 26+ messages in thread

* Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided.
@ 2025-01-22 13:59  Shubham Khanna <[email protected]>
  0 siblings, 0 replies; 26+ messages in thread

From: Shubham Khanna @ 2025-01-22 13:59 UTC (permalink / raw)
  To: PostgreSQL Hackers <[email protected]>

Hi all,

I am writing to propose an enhancement to the pg_createsubscriber
utility that enables it to automatically fetch all non-template
databases from the publisher when no specific databases are specified
by the user. This was an open item from [1] that was planned for
future implementation. The attached patch has the changes for the
same.

[1] - https://www.postgresql.org/message-id/CAA4eK1%2BJpALAokLqxVsQKgo9iFrO-zChfvNXXJMkC8jUgYykBw%40mail.g...

Thanks and regards,
Shubham Khanna.


Attachments:

  [application/octet-stream] v1-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patch (6.8K, ../../CAHv8RjKhA=_h5vAbozzJ1Opnv=KXYQHQ-fJyaMfqfRqPpnC2bA@mail.gmail.com/2-v1-0001-Enhance-pg_createsubscriber-to-fetch-and-append-a.patch)
  download | inline diff:
From 3240552a0e8853cb69191da760cdb34069d83da5 Mon Sep 17 00:00:00 2001
From: Shubham Khanna <[email protected]>
Date: Wed, 22 Jan 2025 12:03:12 +0530
Subject: [PATCH v1] Enhance 'pg_createsubscriber' to fetch and append all
 databases

This patch introduces the ability for 'pg_createsubscriber' to automatically
fetch all non-template databases from the publisher and append them to the
subscription list.
This improvement simplifies the creation of subscriptions for multiple
databases by eliminating the need to specify each database manually.

A new function 'fetch_and_append_all_databases' in 'pg_createsubscriber.c'
that queries the publisher for all databases and adds them to the subscription
options is added.
---
 doc/src/sgml/ref/pg_createsubscriber.sgml     |  7 ++-
 src/bin/pg_basebackup/pg_createsubscriber.c   | 60 +++++++++++++++++++
 .../t/040_pg_createsubscriber.pl              | 52 ++++++++++++++++
 3 files changed, 118 insertions(+), 1 deletion(-)

diff --git a/doc/src/sgml/ref/pg_createsubscriber.sgml b/doc/src/sgml/ref/pg_createsubscriber.sgml
index 26b8e64a4e..0f7c441381 100644
--- a/doc/src/sgml/ref/pg_createsubscriber.sgml
+++ b/doc/src/sgml/ref/pg_createsubscriber.sgml
@@ -94,7 +94,12 @@ PostgreSQL documentation
       <para>
        The name of the database in which to create a subscription.  Multiple
        databases can be selected by writing multiple <option>-d</option>
-       switches.
+       switches. If this option is not provided,
+       <application>pg_createsubscriber</application> will automatically fetch
+       all non-template databases from the publisher and create subscriptions
+       for them. This is particularly useful in scenarios where the user wants
+       to convert a physical standby to a logical subscriber for
+       multiple databases, such as during an upgrade.
       </para>
      </listitem>
     </varlistentry>
diff --git a/src/bin/pg_basebackup/pg_createsubscriber.c b/src/bin/pg_basebackup/pg_createsubscriber.c
index faf18ccf13..db9031c377 100644
--- a/src/bin/pg_basebackup/pg_createsubscriber.c
+++ b/src/bin/pg_basebackup/pg_createsubscriber.c
@@ -106,6 +106,7 @@ static void check_and_drop_existing_subscriptions(PGconn *conn,
 												  const struct LogicalRepInfo *dbinfo);
 static void drop_existing_subscriptions(PGconn *conn, const char *subname,
 										const char *dbname);
+static void fetch_and_append_all_databases(struct CreateSubscriberOptions *opt);
 
 #define	USEC_PER_SEC	1000000
 #define	WAIT_INTERVAL	1		/* 1 second */
@@ -1860,6 +1861,64 @@ enable_subscription(PGconn *conn, const struct LogicalRepInfo *dbinfo)
 	destroyPQExpBuffer(str);
 }
 
+/* Placeholder function to fetch all databases from the publisher */
+static void
+fetch_and_append_all_databases(struct CreateSubscriberOptions *opt)
+{
+	PGconn	   *conn;
+	PGresult   *res;
+	int			num_rows;
+	const char *query = "SELECT datname FROM pg_database WHERE datistemplate = false";
+
+	/* Establish a connection to the PostgreSQL server */
+	conn = PQconnectdb(opt->pub_conninfo_str);
+	/* Check for connection errors */
+	if (PQstatus(conn) != CONNECTION_OK)
+	{
+		pg_log_error("Connection to the PostgreSQL server failed: %s", PQerrorMessage(conn));
+		PQfinish(conn);
+		exit(1);
+	}
+
+	res = PQexec(conn, query);
+
+	/* Check for errors during query execution */
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("Failed to execute query on the PostgreSQL server: %s", PQerrorMessage(conn));
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	/* Process the query result */
+	num_rows = PQntuples(res);
+	for (int i = 0; i < num_rows; i++)
+	{
+		const char *dbname = PQgetvalue(res, i, 0);
+
+		simple_string_list_append(&opt->database_names, dbname);
+		num_dbs++;
+	}
+
+	/* Check if any databases were added */
+	if (opt->database_names.head == NULL)
+	{
+		pg_log_error("no database names could be fetched or specified");
+		pg_log_error_hint("Ensure that databases exist on the publisher or specify a database using --database option.");
+		PQclear(res);
+		PQfinish(conn);
+		exit(1);
+	}
+	else
+	{
+		pg_log_info("databases were fetched and added to the subscription list");
+	}
+
+	PQclear(res);
+	PQfinish(conn);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -2115,6 +2174,7 @@ main(int argc, char **argv)
 							  progname);
 			exit(1);
 		}
+		fetch_and_append_all_databases(&opt);
 	}
 
 	/* Number of object names must match number of databases */
diff --git a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
index c8dbdb7e9b..eed4b18742 100644
--- a/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
+++ b/src/bin/pg_basebackup/t/040_pg_createsubscriber.pl
@@ -368,6 +368,9 @@ command_ok(
 		'--socketdir' => $node_s->host,
 		'--subscriber-port' => $node_s->port,
 		'--replication-slot' => 'replslot1',
+		'--replication-slot' => 'replslot2',
+		'--replication-slot' => 'replslot3',
+		'--replication-slot' => 'replslot4',
 	],
 	'run pg_createsubscriber without --databases');
 
@@ -448,10 +451,59 @@ my $sysid_s = $node_s->safe_psql('postgres',
 	'SELECT system_identifier FROM pg_control_system()');
 ok($sysid_p != $sysid_s, 'system identifier was changed');
 
+# Set up node A as primary
+my $node_a = PostgreSQL::Test::Cluster->new('node_a');
+my $aconnstr = $node_a->connstr;
+$node_a->init(allows_streaming => 'logical');
+$node_a->append_conf('postgresql.conf', 'autovacuum = off');
+$node_a->start;
+
+# Set up node B as standby linking to node A
+$node_a->backup('backup_3');
+my $node_b = PostgreSQL::Test::Cluster->new('node_b');
+$node_b->init_from_backup($node_a, 'backup_3', has_streaming => 1);
+$node_b->append_conf(
+	'postgresql.conf', qq[
+primary_conninfo = '$aconnstr dbname=postgres'
+hot_standby_feedback = on
+max_logical_replication_workers = 5
+]);
+$node_b->set_standby_mode();
+$node_b->start;
+# Ensure there are some user databases on the publisher
+$node_a->safe_psql('postgres', 'CREATE DATABASE db3');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db4');
+$node_a->safe_psql('postgres', 'CREATE DATABASE db5');
+
+$node_b->stop;
+
+# pg_createsubscriber can run without --databases option
+command_ok(
+	[
+		'pg_createsubscriber',
+		'--verbose',
+		'--pgdata' => $node_b->data_dir,
+		'--publisher-server' => $node_a->connstr('postgres'),
+		'--socketdir' => $node_b->host,
+		'--subscriber-port' => $node_b->port,
+	],
+	'run pg_createsubscriber without --databases');
+
+$node_b->start;
+# Verify subscriptions are created for all databases
+my $subscription_count =
+  $node_b->safe_psql('postgres', 'SELECT COUNT (*) FROM pg_subscription');
+is($subscription_count, '4',
+	'Three subscriptions are created for db3, db4, db5');
+
+$node_b->stop;
+
 # clean up
 $node_p->teardown_node;
 $node_s->teardown_node;
 $node_t->teardown_node;
 $node_f->teardown_node;
+$node_a->teardown_node;
+$node_b->teardown_node;
 
 done_testing();
-- 
2.41.0.windows.3



^ permalink  raw  reply  [nested|flat] 26+ messages in thread

end of thread, other threads:[~2025-01-22 13:59 UTC | newest]

Thread overview: 26+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2025-01-22 13:59 Enhance 'pg_createsubscriber' to retrieve databases automatically when no database is provided. Shubham Khanna <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox