public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH] Check connection info in ALTER SUBSCRIPTION
32+ messages / 5 participants
[nested] [flat]

* [PATCH] Check connection info in ALTER SUBSCRIPTION
@ 2017-05-06 12:28 Petr Jelinek <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Petr Jelinek @ 2017-05-06 12:28 UTC (permalink / raw)

---
 src/backend/commands/subscriptioncmds.c    | 5 +++++
 src/test/regress/expected/subscription.out | 4 ++++
 src/test/regress/sql/subscription.sql      | 3 +++
 3 files changed, 12 insertions(+)

diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index afb116a..a9b3f77 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -673,6 +673,11 @@ AlterSubscription(AlterSubscriptionStmt *stmt)
 			}
 
 		case ALTER_SUBSCRIPTION_CONNECTION:
+			/* Load the library providing us libpq calls. */
+			load_file("libpqwalreceiver", false);
+			/* Check the connection info string. */
+			walrcv_check_conninfo(stmt->conninfo);
+
 			values[Anum_pg_subscription_subconninfo - 1] =
 				CStringGetTextDatum(stmt->conninfo);
 			replaces[Anum_pg_subscription_subconninfo - 1] = true;
diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out
index fcf5646..4821113 100644
--- a/src/test/regress/expected/subscription.out
+++ b/src/test/regress/expected/subscription.out
@@ -45,6 +45,10 @@ SET SESSION AUTHORIZATION 'regress_subscription_user2';
 CREATE SUBSCRIPTION testsub2 CONNECTION 'dbname=doesnotexist' PUBLICATION foo WITH (connect = false);
 ERROR:  must be superuser to create subscriptions
 SET SESSION AUTHORIZATION 'regress_subscription_user';
+-- fail - invalid connection string
+ALTER SUBSCRIPTION testsub CONNECTION 'foobar';
+ERROR:  invalid connection string syntax: missing "=" after "foobar" in connection info string
+
 \dRs+
                                          List of subscriptions
   Name   |           Owner           | Enabled | Publication | Synchronous commit |      Conninfo       
diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql
index c27fee5..e5affea 100644
--- a/src/test/regress/sql/subscription.sql
+++ b/src/test/regress/sql/subscription.sql
@@ -38,6 +38,9 @@ SET SESSION AUTHORIZATION 'regress_subscription_user2';
 CREATE SUBSCRIPTION testsub2 CONNECTION 'dbname=doesnotexist' PUBLICATION foo WITH (connect = false);
 SET SESSION AUTHORIZATION 'regress_subscription_user';
 
+-- fail - invalid connection string
+ALTER SUBSCRIPTION testsub CONNECTION 'foobar';
+
 \dRs+
 
 ALTER SUBSCRIPTION testsub SET PUBLICATION testpub2, testpub3 SKIP REFRESH;
-- 
2.7.4


--------------4636A6DFDD524D7B3C589584
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

--------------4636A6DFDD524D7B3C589584--




^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 9a56b3487e..b64c61f742 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1324,7 +1324,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index eccb88b611..ec78de5dd9 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1213,7 +1213,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6469,8 +6472,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index d829e57baa..582b1dd11b 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -422,7 +425,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -433,7 +436,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -479,10 +482,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -552,7 +555,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -579,7 +582,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -631,7 +634,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -658,7 +661,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -792,10 +795,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -808,7 +811,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -832,7 +835,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -968,7 +971,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1022,10 +1025,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1146,7 +1149,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1180,8 +1183,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 97def6f781..ee322ba796 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -189,11 +189,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 33d952f9fd..a0914ddb25 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -578,8 +578,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 7344a7c692..c5881594b7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,7 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -112,7 +112,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -172,7 +172,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 9329a6a6d6..025479431d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1043,7 +1043,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 732d463a49..e9d7d70db7 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.18.4


----Next_Part(Tue_Sep__8_16_35_16_2020_701)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize.
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

XLog reader facility and its callers currently relies on the same
constant XLOG_BLCKSZ. This patch moves the responsibility to determine
the block size to the callers of the facility.
---
 src/backend/access/transam/twophase.c     |  3 +-
 src/backend/access/transam/xlog.c         |  9 ++--
 src/backend/access/transam/xlogreader.c   | 57 ++++++++++++-----------
 src/backend/replication/logical/logical.c |  5 +-
 src/backend/replication/walsender.c       |  4 +-
 src/bin/pg_rewind/parsexlog.c             |  6 +--
 src/bin/pg_waldump/pg_waldump.c           |  2 +-
 src/include/access/xlogreader.h           |  3 +-
 8 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 7d5a96a5f5..9f9e7de253 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1325,7 +1325,8 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b517ca85ad..0822ac70cb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -1222,7 +1222,10 @@ XLogInsertRecord(XLogRecData *rdata,
 			appendBinaryStringInfo(&recordBuf, rdata->data, rdata->len);
 
 		if (!debug_reader)
-			debug_reader = XLogReaderAllocate(wal_segment_size, NULL, NULL);
+		{
+			debug_reader =
+				XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, NULL);
+		}
 
 		if (!debug_reader)
 		{
@@ -6531,8 +6534,8 @@ StartupXLOG(void)
 		OwnLatch(&XLogCtl->recoveryWakeupLatch);
 
 	/* Set up XLOG reader facility */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ba3e4868f7..e5baf13086 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -72,7 +72,7 @@ report_invalid_record(XLogReaderState *state, const char *fmt,...)
  * Returns NULL if the xlogreader couldn't be allocated.
  */
 XLogReaderState *
-XLogReaderAllocate(int wal_segment_size, const char *waldir,
+XLogReaderAllocate(int wal_segment_size, int page_size, const char *waldir,
 				   WALSegmentCleanupCB cleanup_cb)
 {
 	XLogReaderState *state;
@@ -87,6 +87,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	state->cleanup_cb = cleanup_cb;
 
 	state->max_block_id = -1;
+	state->pageSize = page_size;
 
 	/*
 	 * Permanently allocate readBuf.  We do it this way, rather than just
@@ -95,7 +96,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	 * isn't guaranteed to have any particular alignment, whereas
 	 * palloc_extended() will provide MAXALIGN'd storage.
 	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
+	state->readBuf = (char *) palloc_extended(state->pageSize,
 											  MCXT_ALLOC_NO_OOM);
 	if (!state->readBuf)
 	{
@@ -163,7 +164,7 @@ XLogReaderFree(XLogReaderState *state)
  * readRecordBufSize is set to the new buffer size.
  *
  * To avoid useless small increases, round its size to a multiple of
- * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start
+ * page size, and make sure it's at least 5*Max(BLCKSZ, <page size>) to start
  * with.  (That is enough for all "normal" records, but very large commit or
  * abort records might need more space.)
  */
@@ -172,8 +173,8 @@ allocate_recordbuf(XLogReaderState *state, uint32 reclength)
 {
 	uint32		newSize = reclength;
 
-	newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ);
-	newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ));
+	newSize += state->pageSize - (newSize % state->pageSize);
+	newSize = Max(newSize, 5 * Max(BLCKSZ, state->pageSize));
 
 #ifndef FRONTEND
 
@@ -238,6 +239,8 @@ void
 XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr)
 {
 	Assert(!XLogRecPtrIsInvalid(RecPtr));
+	Assert(state->readBuf != NULL &&
+		   state->readBuf == (void *) MAXALIGN(state->readBuf));
 
 	ResetDecoder(state);
 
@@ -366,15 +369,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			XLogPageHeader pageHeader;
 
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
 			/*
 			 * Check if we have enough data. For the first record in the page,
 			 * the requesting length doesn't contain page header.
 			 */
 			if (XLogNeedData(state, targetPagePtr,
-							 Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+							 Min(targetRecOff + SizeOfXLogRecord, state->pageSize),
 							 targetRecOff != 0))
 				return XLREAD_NEED_DATA;
 
@@ -421,7 +424,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * verified that we got the whole header.
 			 */
 			prec = (XLogRecord *) (state->readBuf +
-								   state->ReadRecPtr % XLOG_BLCKSZ);
+								   state->ReadRecPtr % state->pageSize);
 			total_len = prec->xl_tot_len;
 
 			/*
@@ -432,7 +435,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * ensure that we enter the XLREAD_CONTINUATION state below;
 			 * otherwise we might fail to apply ValidXLogRecordHeader at all.
 			 */
-			if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord)
+			if (targetRecOff <= state->pageSize - SizeOfXLogRecord)
 			{
 				if (!ValidXLogRecordHeader(state, state->ReadRecPtr,
 										   state->PrevRecPtr, prec))
@@ -477,10 +480,10 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 			 * available
 			 */
 			targetPagePtr =
-				state->ReadRecPtr - (state->ReadRecPtr % XLOG_BLCKSZ);
-			targetRecOff = state->ReadRecPtr % XLOG_BLCKSZ;
+				state->ReadRecPtr - (state->ReadRecPtr % state->pageSize);
+			targetRecOff = state->ReadRecPtr % state->pageSize;
 
-			request_len = Min(targetRecOff + total_len, XLOG_BLCKSZ);
+			request_len = Min(targetRecOff + total_len, state->pageSize);
 			record_len = request_len - targetRecOff;
 
 			/* ReadRecPtr contains page header */
@@ -549,7 +552,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 
 			/* Calculate pointer to beginning of next page */
 			state->recordContRecPtr = state->ReadRecPtr + record_len;
-			Assert(state->recordContRecPtr % XLOG_BLCKSZ == 0);
+			Assert(state->recordContRecPtr % state->pageSize == 0);
 
 			state->readRecordState = XLREAD_CONTINUATION;
 		}
@@ -576,7 +579,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				/* this request contains page header */
 				Assert (targetPagePtr != 0);
 				if (XLogNeedData(state, targetPagePtr,
-								 Min(state->recordRemainLen, XLOG_BLCKSZ),
+								 Min(state->recordRemainLen, state->pageSize),
 								 false))
 					return XLREAD_NEED_DATA;
 
@@ -628,7 +631,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				Assert(state->readLen >= pageHeaderSize);
 
 				contdata = (char *) state->readBuf + pageHeaderSize;
-				record_len = XLOG_BLCKSZ - pageHeaderSize;
+				record_len = state->pageSize - pageHeaderSize;
 				if (pageHeader->xlp_rem_len < record_len)
 					record_len = pageHeader->xlp_rem_len;
 
@@ -655,7 +658,7 @@ XLogReadRecord(XLogReaderState *state, XLogRecord **record, char **errormsg)
 				}
 
 				/* Calculate pointer to beginning of next page, and continue */
-				state->recordContRecPtr += XLOG_BLCKSZ;
+				state->recordContRecPtr += state->pageSize;
 			}
 
 			/* targetPagePtr is pointing the last-read page here */
@@ -789,10 +792,10 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 				XLogPageHeaderSize((XLogPageHeader) state->readBuf);
 
 			addLen = pageHeaderSize;
-			if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+			if (reqLen + pageHeaderSize <= state->pageSize)
 				addLen = pageHeaderSize;
 			else
-				addLen = XLOG_BLCKSZ - reqLen;
+				addLen = state->pageSize - reqLen;
 
 			Assert(addLen >= 0);
 		}
@@ -805,7 +808,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 	/* Data is not in our buffer, request the caller for it. */
 	XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
 	targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
-	Assert((pageptr % XLOG_BLCKSZ) == 0);
+	Assert((pageptr % state->pageSize) == 0);
 
 	/*
 	 * Every time we request to load new data of a page to the caller, even if
@@ -829,7 +832,7 @@ XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
 		 * will not come back here, but will request the actual target page.
 		 */
 		state->readPagePtr = pageptr - targetPageOff;
-		state->readLen = XLOG_BLCKSZ;
+		state->readLen = state->pageSize;
 		return true;
 	}
 
@@ -962,7 +965,7 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	int32		offset;
 	XLogPageHeader hdr = (XLogPageHeader) phdr;
 
-	Assert((recptr % XLOG_BLCKSZ) == 0);
+	Assert((recptr % state->pageSize) == 0);
 
 	XLByteToSeg(recptr, segno, state->segcxt.ws_segsize);
 	offset = XLogSegmentOffset(recptr, state->segcxt.ws_segsize);
@@ -1016,10 +1019,10 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
 								  "WAL file is from different database system: incorrect segment size in page header");
 			return false;
 		}
-		else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
+		else if (longhdr->xlp_xlog_blcksz != state->pageSize)
 		{
 			report_invalid_record(state,
-								  "WAL file is from different database system: incorrect XLOG_BLCKSZ in page header");
+								  "WAL file is from different database system: incorrect block size in page header");
 			return false;
 		}
 	}
@@ -1140,7 +1143,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 		 * XLogNeedData() is prepared to handle that and will read at
 		 * least short page-header worth of data
 		 */
-		targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
+		targetRecOff = tmpRecPtr % state->pageSize;
 
 		/* scroll back to page boundary */
 		targetPagePtr = tmpRecPtr - targetRecOff;
@@ -1174,8 +1177,8 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr,
 			 *
 			 * Note that record headers are MAXALIGN'ed
 			 */
-			if (MAXALIGN(header->xlp_rem_len) >= (XLOG_BLCKSZ - pageHeaderSize))
-				tmpRecPtr = targetPagePtr + XLOG_BLCKSZ;
+			if (MAXALIGN(header->xlp_rem_len) >= (state->pageSize - pageHeaderSize))
+				tmpRecPtr = targetPagePtr + state->pageSize;
 			else
 			{
 				/*
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 9cbd8afa49..f2dd495310 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -199,11 +199,14 @@ StartupDecodingContext(List *output_plugin_options,
 
 	ctx->slot = slot;
 
-	ctx->reader = XLogReaderAllocate(wal_segment_size, NULL, cleanup_cb);
+	ctx->reader =
+		XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ, NULL, cleanup_cb);
+
 	if (!ctx->reader)
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 062093727d..8e95600ef7 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -579,8 +579,8 @@ StartReplication(StartReplicationCmd *cmd)
 				 errmsg("IDENTIFY_SYSTEM has not been run before START_REPLICATION")));
 
 	/* create xlogreader for physical replication */
-	xlogreader =
-		XLogReaderAllocate(wal_segment_size, NULL, wal_segment_close);
+	xlogreader = XLogReaderAllocate(wal_segment_size, XLOG_BLCKSZ,
+									NULL, wal_segment_close);
 
 	if (!xlogreader)
 		ereport(ERROR,
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 712c85281c..8a6209b061 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,7 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	XLogReaderState *xlogreader;
 	char	   *errormsg;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
@@ -121,7 +121,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	char	   *errormsg;
 	XLogRecPtr	endptr;
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
@@ -181,7 +181,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 			forkptr += SizeOfXLogShortPHD;
 	}
 
-	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
+	xlogreader = XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
 
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index f514de6b13..b635537659 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1041,7 +1041,7 @@ main(int argc, char **argv)
 
 	/* we have everything we need, start reading */
 	xlogreader_state =
-		XLogReaderAllocate(WalSegSz, waldir, WALDumpCloseSegment);
+		XLogReaderAllocate(WalSegSz, XLOG_BLCKSZ, waldir, WALDumpCloseSegment);
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index b9dad7ee51..1853516333 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -151,6 +151,7 @@ struct XLogReaderState
 	 *  ----------------------------------------
 	 */
 	/* variables to communicate with page reader */
+	int			pageSize;		/* size of a page */
 	XLogRecPtr	readPagePtr;	/* page pointer to read */
 	int32		readLen;		/* bytes requested to reader, or actual bytes
 								 * read by reader, which must be larger than
@@ -243,7 +244,7 @@ struct XLogReaderState
 
 /* Get a new XLogReader */
 extern XLogReaderState *XLogReaderAllocate(int wal_segment_size,
-										   const char *waldir,
+										   int page_size, const char *waldir,
 										   WALSegmentCleanupCB cleanup_cb);
 
 /* Free an XLogReader */
-- 
2.27.0


----Next_Part(Thu_Mar__4_11_28_53_2021_014)----





^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Crash in add_paths_to_append_rel
@ 2023-10-09 09:45 Richard Guo <[email protected]>
  2023-10-09 11:35 ` Re: Crash in add_paths_to_append_rel David Rowley <[email protected]>
  2023-10-10 03:52 ` Re: Crash in add_paths_to_append_rel David Rowley <[email protected]>
  0 siblings, 2 replies; 32+ messages in thread

From: Richard Guo @ 2023-10-09 09:45 UTC (permalink / raw)
  To: pgsql-hackers

I came across a crash in add_paths_to_append_rel() with the query below.

create table t (a int);

create table inh (b int);
create table inh_child() inherits(inh);

explain (costs off)
select * from t left join lateral (select t.a from inh) on true limit 1;
server closed the connection unexpectedly

It was introduced by commit a8a968a8 where we referenced
cheapest_startup_path->param_info without first checking that we have a
valid cheapest_startup_path.  We do have checked that the pathlist is
not empty, but that does not imply that the cheapest_startup_path is not
NULL.  Normally only unparameterized paths are considered candidates for
cheapest_startup_path, because startup cost does not have too much value
for parameterized paths since they are on the inside of a nestloop.  So
if there are no unparameterized paths, the cheapest_startup_path will be
NULL.  That is the case in the repro query above.  Because of the
lateral reference within PHV, the child rels of 'inh' do not have
unparameterized paths, so their cheapest_startup_path is NULL.

I think we should explicitly check that cheapest_startup_path is not
NULL here.  Doing that seems to make the check of pathlist not being
empty unnecessary.  Besides, I think we do not need to check that
cheapest_startup_path->param_info is NULL, since cheapest_startup_path
must be unparameterized path.  Maybe we can use an Assert instead.

Attached is my proposed fix.

Thanks
Richard


Attachments:

  [application/octet-stream] v1-0001-Don-t-assume-that-cheapest_startup_path-exists.patch (3.4K, ../../CAMbWs49w3t03V69XhdCuw+GDwivny4uQUxrkVp6Gejaspt0wMQ@mail.gmail.com/3-v1-0001-Don-t-assume-that-cheapest_startup_path-exists.patch)
  download | inline diff:
From 5c555fb67037d34f3a90be6052cb2fa26f34769a Mon Sep 17 00:00:00 2001
From: Richard Guo <[email protected]>
Date: Mon, 9 Oct 2023 17:10:59 +0800
Subject: [PATCH v1] Don't assume that cheapest_startup_path exists

Normally only unparameterized paths are considered candidates for
cheapest_startup_path, because startup cost does not have too much value
for parameterized paths since they are on the inside of a nestloop.  So
if there are no unparameterized paths, the cheapest_startup_path will be
NULL.  In add_paths_to_append_rel() we should not assume that
cheapest_startup_path always exists.
---
 src/backend/optimizer/path/allpaths.c | 12 ++++++++++--
 src/test/regress/expected/union.out   | 16 ++++++++++++++++
 src/test/regress/sql/union.sql        |  9 ++++++++-
 3 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 7af001feaa..5c4e3695e5 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -1352,12 +1352,20 @@ add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel,
 		 * When the planner is considering cheap startup plans, we'll also
 		 * collect all the cheapest_startup_paths and build an AppendPath
 		 * containing those as subpaths.
+		 *
+		 * Only unparameterized paths are considered candidates for
+		 * cheapest_startup_path, so that will be NULL if there are no
+		 * unparameterized paths.  We have to explicitly check that we have a
+		 * valid cheapest_startup_path.
 		 */
-		if (rel->consider_startup && childrel->pathlist != NIL &&
-			childrel->cheapest_startup_path->param_info == NULL)
+		if (rel->consider_startup && childrel->cheapest_startup_path != NULL)
+		{
+			/* cheapest_startup_path must be unparameterized path. */
+			Assert(childrel->cheapest_startup_path->param_info == NULL);
 			accumulate_append_subpath(childrel->cheapest_startup_path,
 									  &startup_subpaths,
 									  NULL);
+		}
 		else
 			startup_subpaths_valid = false;
 
diff --git a/src/test/regress/expected/union.out b/src/test/regress/expected/union.out
index f046e522de..a4619cb014 100644
--- a/src/test/regress/expected/union.out
+++ b/src/test/regress/expected/union.out
@@ -1453,3 +1453,19 @@ inner join tenk2 t2 on t1.tenthous = t2.tenthous
          ->  Result
 (8 rows)
 
+-- Ensure there is no problem if cheapest_startup_path is NULL
+explain (costs off)
+select * from tenk1 t1
+left join lateral
+	(select t1.tenthous from tenk2 t2 union all (values(1)))
+on true limit 1;
+                            QUERY PLAN                             
+-------------------------------------------------------------------
+ Limit
+   ->  Nested Loop Left Join
+         ->  Seq Scan on tenk1 t1
+         ->  Append
+               ->  Index Only Scan using tenk2_hundred on tenk2 t2
+               ->  Result
+(6 rows)
+
diff --git a/src/test/regress/sql/union.sql b/src/test/regress/sql/union.sql
index d65ca9f86d..213309e226 100644
--- a/src/test/regress/sql/union.sql
+++ b/src/test/regress/sql/union.sql
@@ -550,4 +550,11 @@ explain (costs off)
 select t1.unique1 from tenk1 t1
 inner join tenk2 t2 on t1.tenthous = t2.tenthous
    union all
-(values(1)) limit 1;
\ No newline at end of file
+(values(1)) limit 1;
+
+-- Ensure there is no problem if cheapest_startup_path is NULL
+explain (costs off)
+select * from tenk1 t1
+left join lateral
+	(select t1.tenthous from tenk2 t2 union all (values(1)))
+on true limit 1;
-- 
2.31.0



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Crash in add_paths_to_append_rel
  2023-10-09 09:45 Crash in add_paths_to_append_rel Richard Guo <[email protected]>
@ 2023-10-09 11:35 ` David Rowley <[email protected]>
  2023-10-10 05:49   ` Re: Crash in add_paths_to_append_rel Richard Guo <[email protected]>
  1 sibling, 1 reply; 32+ messages in thread

From: David Rowley @ 2023-10-09 11:35 UTC (permalink / raw)
  To: Richard Guo <[email protected]>; +Cc: pgsql-hackers

On Mon, 9 Oct 2023 at 22:49, Richard Guo <[email protected]> wrote:
> I came across a crash in add_paths_to_append_rel() with the query below.

> It was introduced by commit a8a968a8 where we referenced
> cheapest_startup_path->param_info without first checking that we have a
> valid cheapest_startup_path.

Thank you for the report.

Since you've managed to attribute this to a specific commit, for the
future, I think instead of opening a new thread, it would be more
useful to communicate the issue on the thread that's linked in the
commit message.

I will look at this tomorrow.

David






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Crash in add_paths_to_append_rel
  2023-10-09 09:45 Crash in add_paths_to_append_rel Richard Guo <[email protected]>
  2023-10-09 11:35 ` Re: Crash in add_paths_to_append_rel David Rowley <[email protected]>
@ 2023-10-10 05:49   ` Richard Guo <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Richard Guo @ 2023-10-10 05:49 UTC (permalink / raw)
  To: David Rowley <[email protected]>; +Cc: pgsql-hackers

On Mon, Oct 9, 2023 at 7:35 PM David Rowley <[email protected]> wrote:

> Since you've managed to attribute this to a specific commit, for the
> future, I think instead of opening a new thread, it would be more
> useful to communicate the issue on the thread that's linked in the
> commit message.


Ah, yes, I should have done it that way, sorry.

Thanks
Richard


^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Crash in add_paths_to_append_rel
  2023-10-09 09:45 Crash in add_paths_to_append_rel Richard Guo <[email protected]>
@ 2023-10-10 03:52 ` David Rowley <[email protected]>
  2023-10-10 04:48   ` Re: Crash in add_paths_to_append_rel Andy Fan <[email protected]>
  2023-10-10 05:50   ` Re: Crash in add_paths_to_append_rel Richard Guo <[email protected]>
  1 sibling, 2 replies; 32+ messages in thread

From: David Rowley @ 2023-10-10 03:52 UTC (permalink / raw)
  To: Richard Guo <[email protected]>; +Cc: pgsql-hackers

On Mon, 9 Oct 2023 at 22:49, Richard Guo <[email protected]> wrote:
> I came across a crash in add_paths_to_append_rel() with the query below.

> It was introduced by commit a8a968a8 where we referenced
> cheapest_startup_path->param_info without first checking that we have a
> valid cheapest_startup_path.

I pushed this with just minor adjustments to the comments you wrote. I
didn't feel the need to reiterate on the set_cheapest() comments.

David






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Crash in add_paths_to_append_rel
  2023-10-09 09:45 Crash in add_paths_to_append_rel Richard Guo <[email protected]>
  2023-10-10 03:52 ` Re: Crash in add_paths_to_append_rel David Rowley <[email protected]>
@ 2023-10-10 04:48   ` Andy Fan <[email protected]>
  1 sibling, 0 replies; 32+ messages in thread

From: Andy Fan @ 2023-10-10 04:48 UTC (permalink / raw)
  To: David Rowley <[email protected]>; +Cc: Richard Guo <[email protected]>; pgsql-hackers

On Tue, Oct 10, 2023 at 11:52 AM David Rowley <[email protected]> wrote:

> On Mon, 9 Oct 2023 at 22:49, Richard Guo <[email protected]> wrote:
> > I came across a crash in add_paths_to_append_rel() with the query below.
>
> > It was introduced by commit a8a968a8 where we referenced
> > cheapest_startup_path->param_info without first checking that we have a
> > valid cheapest_startup_path.
>
> I pushed this with just minor adjustments to the comments you wrote. I
> didn't feel the need to reiterate on the set_cheapest() comments.
>
>
 Thanks Richard for the report and also Thanks you David for the push!

-- 
Best Regards
Andy Fan


^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Crash in add_paths_to_append_rel
  2023-10-09 09:45 Crash in add_paths_to_append_rel Richard Guo <[email protected]>
  2023-10-10 03:52 ` Re: Crash in add_paths_to_append_rel David Rowley <[email protected]>
@ 2023-10-10 05:50   ` Richard Guo <[email protected]>
  1 sibling, 0 replies; 32+ messages in thread

From: Richard Guo @ 2023-10-10 05:50 UTC (permalink / raw)
  To: David Rowley <[email protected]>; +Cc: pgsql-hackers

On Tue, Oct 10, 2023 at 11:52 AM David Rowley <[email protected]> wrote:

> On Mon, 9 Oct 2023 at 22:49, Richard Guo <[email protected]> wrote:
> > I came across a crash in add_paths_to_append_rel() with the query below.
>
> > It was introduced by commit a8a968a8 where we referenced
> > cheapest_startup_path->param_info without first checking that we have a
> > valid cheapest_startup_path.
>
> I pushed this with just minor adjustments to the comments you wrote. I
> didn't feel the need to reiterate on the set_cheapest() comments.


Thanks for pushing!

Thanks
Richard


^ permalink  raw  reply  [nested|flat] 32+ messages in thread

end of thread, other threads:[~2023-10-10 05:50 UTC | newest]

Thread overview: 32+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2017-05-06 12:28 [PATCH] Check connection info in ALTER SUBSCRIPTION Petr Jelinek <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v16 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v17 4/4] Allow xlogreader to use different xlog blocksize. Kyotaro Horiguchi <[email protected]>
2023-10-09 09:45 Crash in add_paths_to_append_rel Richard Guo <[email protected]>
2023-10-09 11:35 ` Re: Crash in add_paths_to_append_rel David Rowley <[email protected]>
2023-10-10 05:49   ` Re: Crash in add_paths_to_append_rel Richard Guo <[email protected]>
2023-10-10 03:52 ` Re: Crash in add_paths_to_append_rel David Rowley <[email protected]>
2023-10-10 04:48   ` Re: Crash in add_paths_to_append_rel Andy Fan <[email protected]>
2023-10-10 05:50   ` Re: Crash in add_paths_to_append_rel Richard Guo <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox